Skip to content

bigger files in sqlite #165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
krisoft wants to merge 1 commit into
base: purgatory
Choose a base branch
from
Open

bigger files in sqlite #165

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 133 additions & 21 deletions Android/G3MAndroidSDK/src/org/glob3/mobile/specific/SQLiteStorage_Android.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@

package org.glob3.mobile.specific;

import java.io.ByteArrayOutputStream;
import java.io.File;
import android.content.ContentValues;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.util.Log;

import org.glob3.mobile.generated.G3MContext;
import org.glob3.mobile.generated.GTask;
Expand All @@ -16,19 +21,23 @@
import org.glob3.mobile.generated.TimeInterval;
import org.glob3.mobile.generated.URL;

import android.content.ContentValues;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;


public final class SQLiteStorage_Android
extends
IStorage {

private static final long DB_RAW_DATA_EXTERNAL_THRESHOLD = 1024 * 1024; // 1 MiB
private static final String DB_EXTERNAL_TAG = "EXT:";
Copy link
Copy Markdown
Member

@DiegoGomezDeck DiegoGomezDeck Jan 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's make DB_RAW_DATA_EXTERNAL_THRESHOLD an instance field and add a constructor where the threshold could be defined when creating the g3m widget.

private static final Charset UTF8 = Charset.forName("UTF-8");

private static final String[] COLUMNS = new String[] { "contents", "expiration" };
private static final String SELECTION = "name = ?";

Expand All @@ -41,6 +50,7 @@ public final class SQLiteStorage_Android

private final BitmapFactory.Options _options;
private final byte[] _temp_storage = new byte[128 * 1024];
private final String _documentsDirectory;


private class MySQLiteOpenHelper
Expand Down Expand Up @@ -81,13 +91,7 @@ public void onUpgrade(final SQLiteDatabase db,


private String getPath() {
File f = _androidContext.getExternalCacheDir();
if ((f == null) || !f.exists()) {
f = _androidContext.getCacheDir();
}
final String documentsDirectory = f.getAbsolutePath();

final File f2 = new File(new File(documentsDirectory), _databaseName);
final File f2 = new File(new File(_documentsDirectory), _databaseName);

final String path = f2.getAbsolutePath();
Log.d("SQLiteStorage_Android", "Creating DB in " + path);
Expand All @@ -98,8 +102,14 @@ private String getPath() {

public SQLiteStorage_Android(final String path,
final android.content.Context context) {
_androidContext = context;
File f = _androidContext.getExternalCacheDir();
if ((f == null) || !f.exists()) {
f = _androidContext.getCacheDir();
}
_documentsDirectory = f.getAbsolutePath();

_databaseName = path;
_androidContext = context;

_dbHelper = new MySQLiteOpenHelper(context, getPath());
_writeDB = _dbHelper.getWritableDatabase();
Expand Down Expand Up @@ -135,21 +145,88 @@ public void run(final G3MContext context) {
}
}

private boolean writeFileToCacheFolder(final String fileName, final byte[] content) {
final File f2 = new File(new File(_documentsDirectory), fileName);
FileOutputStream fout = null;
try {
fout = new FileOutputStream(f2);
fout.write(content);
fout.flush();
return true;
} catch (FileNotFoundException e) {
e.printStackTrace();
return false;
} catch (IOException e) {
e.printStackTrace();
return false;
} finally {
if (fout != null) {
try {
fout.close();
} catch (IOException e1) {
e1.printStackTrace();
}
}
}
}

private byte[] readFileFromCacheFolder(final String fileName) {
final File f2 = new File(new File(_documentsDirectory), fileName);
FileInputStream fin = null;
try {
fin = new FileInputStream(f2);
final int avail = fin.available();
if (avail == 0) {
Copy link
Copy Markdown
Member

@DiegoGomezDeck DiegoGomezDeck Jan 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm quiet sure this use of available() method is wrong.

Take a look to this:
https://docs.oracle.com/javase/7/docs/api/java/io/FileInputStream.html#available()

Copy link
Copy Markdown
Member

@DiegoGomezDeck DiegoGomezDeck Jan 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You need something like:

   public static byte[] readAll(final InputStream is) throws IOException {
      final ByteArrayOutputStream result = new ByteArrayOutputStream();
      final byte[] buffer = new byte[16384];
      int read;
      while ((read = is.read(buffer, 0, buffer.length)) != -1) {
         result.write(buffer, 0, read);
      }
      result.flush();
      return result.toByteArray();
   }


   public static byte[] readAll(final File file) throws IOException {
      try (final FileInputStream is = new FileInputStream(file)) {
         return readAll(is);
      }
   }

Copy link
Copy Markdown
Author

@krisoft krisoft Jan 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you very much @DiegoGomezDeck! Looks very much like you are right. I will consult with @DrakkLord and one of us will fix this.

return null;
}
final byte[] r = new byte[avail];
if (fin.read(r) != avail) {
return null;
}
return r;
} catch (FileNotFoundException e) {
e.printStackTrace();
return null;
} catch (IOException e) {
e.printStackTrace();
return null;
} finally {
if (fin != null) {
try {
fin.close();
} catch (IOException e1) {
e1.printStackTrace();
}
}
}
}

private synchronized void rawSave(final String table,
final String name,
final byte[] contents,
final TimeInterval timeToExpires) {

final boolean external = contents.length >= DB_RAW_DATA_EXTERNAL_THRESHOLD;
final String externalFileName = external ? (table + name).hashCode() + ".raw_cache" : null;

Copy link
Copy Markdown
Member

@DiegoGomezDeck DiegoGomezDeck Jan 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using .hashCode() here as a file-name will produces collisions of file names.

final ContentValues values = new ContentValues(3);
values.put("name", name);
values.put("contents", contents);
values.put("contents",
external ? (DB_EXTERNAL_TAG + externalFileName).getBytes(UTF8) : contents);
final long expiration = System.currentTimeMillis() + timeToExpires.milliseconds();
values.put("expiration", Long.toString(expiration));

if (_writeDB == null) {
ILogger.instance().logError("SQL: Can't write " + table + " in database \"%s\". _writeDB not available\n", _databaseName);
}
else {
if (external) {
if (!writeFileToCacheFolder(externalFileName, contents)) {
ILogger.instance().logError("SQL: Can't write external file for cache \"%s\"\n", name);
return;
}
}

final long rowID = _writeDB.insertWithOnConflict(table, null, values, SQLiteDatabase.CONFLICT_REPLACE);
if (rowID == -1) {
ILogger.instance().logError("SQL: Can't write " + table + " in database \"%s\"\n", _databaseName);
Expand All @@ -165,7 +242,7 @@ public IByteBufferResult readBuffer(final URL url,
boolean expired = false;
final String name = url._path;

final Cursor cursor = _readDB.query( //
final Cursor cursor = _readDB.query( //
"buffer2", //
COLUMNS, //
SELECTION, //
Expand All @@ -180,7 +257,23 @@ public IByteBufferResult readBuffer(final URL url,

expired = (expirationInterval <= System.currentTimeMillis());
if (!expired || readExpired) {
buffer = new ByteBuffer_Android(data);
if (data.length > DB_EXTERNAL_TAG.length()) {
final String extTag = new String(data, 0, DB_EXTERNAL_TAG.length(), UTF8);
if (!extTag.isEmpty() && DB_EXTERNAL_TAG.equalsIgnoreCase(extTag)) {
final byte[] cacheData = readFileFromCacheFolder(
new String(data, DB_EXTERNAL_TAG.length(), data.length - DB_EXTERNAL_TAG.length(), UTF8));
if (cacheData == null || cacheData.length == 0) {
ILogger.instance()
.logError("SQL: Can't read external file for cache \"%s\"\n", name);
cursor.close();
return new IByteBufferResult(null, expired);
}
buffer = new ByteBuffer_Android(cacheData);
}
}
if (buffer == null) {
buffer = new ByteBuffer_Android(data);
}
}
}
cursor.close();
Expand Down Expand Up @@ -251,8 +344,27 @@ public IImageResult readImage(final URL url,

expired = (expirationInterval <= System.currentTimeMillis());
if (!expired || readExpired) {
Bitmap bitmap = null;

if (data.length > DB_EXTERNAL_TAG.length()) {
final String extTag = new String(data, 0, DB_EXTERNAL_TAG.length(), UTF8);
if (!extTag.isEmpty() && DB_EXTERNAL_TAG.equalsIgnoreCase(extTag)) {
final byte[] cacheData = readFileFromCacheFolder(
new String(data, DB_EXTERNAL_TAG.length(), data.length - DB_EXTERNAL_TAG.length(), UTF8));
if (cacheData == null || cacheData.length == 0) {
ILogger.instance()
.logError("SQL: Can't read external file for cache \"%s\"\n", name);
cursor.close();
return new IImageResult(null, expired);
}
bitmap = BitmapFactory.decodeByteArray(cacheData, 0, cacheData.length, _options);
}
}

// final long start = System.currentTimeMillis();
final Bitmap bitmap = BitmapFactory.decodeByteArray(data, 0, data.length, _options);
if (bitmap == null) {
bitmap = BitmapFactory.decodeByteArray(data, 0, data.length, _options);
}
// ILogger.instance().logInfo("CACHE: Bitmap parsed in " + (System.currentTimeMillis() - start) + "ms");

if (bitmap == null) {
Expand Down