box getAll/getMany - large array support on 32-bit systems

Author: Ivan Dlugos

lib/src/bindings/bindings.dart

@@ -69,6 +69,10 @@ class _ObjectBoxBindings {
   int Function(Pointer<Void> box, int id, Pointer<Pointer<Uint8>> data, Pointer<IntPtr> size) obx_box_get;
   Pointer<OBX_bytes_array> Function(Pointer<Void> box, Pointer<OBX_id_array> ids) obx_box_get_many;
   Pointer<OBX_bytes_array> Function(Pointer<Void> box) obx_box_get_all;
+  int Function(Pointer<Void> box, Pointer<OBX_id_array> ids, Pointer<NativeFunction<obx_data_visitor_native_t>> visitor,
+      Pointer<Void> user_data) obx_box_visit_many;
+  int Function(Pointer<Void> box, Pointer<NativeFunction<obx_data_visitor_native_t>> visitor, Pointer<Void> user_data)
+      obx_box_visit_all;
   int Function(Pointer<Void> box, int id_or_zero) obx_box_id_for_put;
   int Function(Pointer<Void> box, int count, Pointer<Uint64> out_first_id) obx_box_ids_for_put;
   int Function(Pointer<Void> box, int id, Pointer<Uint8> data, int size, int mode) obx_box_put;
@@ -217,6 +221,8 @@ class _ObjectBoxBindings {
     obx_box_get = _fn<obx_box_get_native_t>("obx_box_get").asFunction();
     obx_box_get_many = _fn<obx_box_get_many_native_t>("obx_box_get_many").asFunction();
     obx_box_get_all = _fn<obx_box_get_all_native_t>("obx_box_get_all").asFunction();
+    obx_box_visit_many = _fn<obx_box_visit_many_native_t>("obx_box_visit_many").asFunction();
+    obx_box_visit_all = _fn<obx_box_visit_all_native_t>("obx_box_visit_all").asFunction();
     obx_box_id_for_put = _fn<obx_box_id_for_put_native_t>("obx_box_id_for_put").asFunction();
     obx_box_ids_for_put = _fn<obx_box_ids_for_put_native_t>("obx_box_ids_for_put").asFunction();
     obx_box_put = _fn<obx_box_put_native_t>("obx_box_put").asFunction();

lib/src/bindings/signatures.dart

@@ -61,6 +61,10 @@ typedef obx_box_get_native_t = Int32 Function(
     Pointer<Void> box, Uint64 id, Pointer<Pointer<Uint8>> data, Pointer<IntPtr> size);
 typedef obx_box_get_many_native_t = Pointer<OBX_bytes_array> Function(Pointer<Void> box, Pointer<OBX_id_array> ids);
 typedef obx_box_get_all_native_t = Pointer<OBX_bytes_array> Function(Pointer<Void> box);
+typedef obx_box_visit_many_native_t = Int32 Function(Pointer<Void> box, Pointer<OBX_id_array> ids,
+    Pointer<NativeFunction<obx_data_visitor_native_t>> visitor, Pointer<Void> user_data);
+typedef obx_box_visit_all_native_t = Int32 Function(
+    Pointer<Void> box, Pointer<NativeFunction<obx_data_visitor_native_t>> visitor, Pointer<Void> user_data);
 typedef obx_box_id_for_put_native_t = Uint64 Function(Pointer<Void> box, Uint64 id_or_zero);
 typedef obx_box_ids_for_put_native_t = Int32 Function(Pointer<Void> box, Uint64 count, Pointer<Uint64> out_first_id);
 typedef obx_box_put_native_t = Int32 Function(

lib/src/box.dart

@@ -5,6 +5,7 @@ import 'common.dart';
 import "store.dart";
 import "bindings/bindings.dart";
 import "bindings/constants.dart";
+import "bindings/data_visitor.dart";
 import "bindings/flatbuffers.dart";
 import "bindings/helpers.dart";
 import "bindings/structs.dart";
@@ -24,8 +25,9 @@ class Box<T> {
   ModelEntity _modelEntity;
   ObjectReader<T> _entityReader;
   OBXFlatbuffersManager<T> _fbManager;
+  final bool _supportsBytesArrays;
 
-  Box(this._store) {
+  Box(this._store) : _supportsBytesArrays = bindings.obx_supports_bytes_array() == 1 {
     EntityDefinition<T> entityDefs = _store.entityDef<T>();
     _modelEntity = entityDefs.model;
     _entityReader = entityDefs.reader;
@@ -160,34 +162,63 @@ class Box<T> {
     }
   }
 
-  List<T> _getMany(bool allowMissing, Pointer<OBX_bytes_array> Function() cCall) {
+  List<T> _getMany(
+      bool allowMissing, Pointer<OBX_bytes_array> Function() cGetArray, void Function(DataVisitor) cVisit) {
     return _store.runInTransaction(TxMode.Read, () {
-      final bytesArray = cCall();
+      if (_supportsBytesArrays) {
+        final bytesArray = cGetArray();
+        try {
+          return _fbManager.unmarshalArray(bytesArray, allowMissing: false);
+        } finally {
+          bindings.obx_bytes_array_free(bytesArray);
+        }
+      }
+
+      final results = <T>[];
+      final visitor = DataVisitor((Pointer<Uint8> dataPtr, int length) {
+        if (dataPtr == null || dataPtr.address == 0 || length == 0) {
+          if (allowMissing) {
+            results.add(null);
+            return true;
+          } else {
+            throw Exception('Object not found');
+          }
+        }
+        final bytes = dataPtr.asTypedList(length);
+        results.add(_fbManager.unmarshal(bytes));
+        return true;
+      });
+
       try {
-        return _fbManager.unmarshalArray(bytesArray, allowMissing: allowMissing);
+        cVisit(visitor);
       } finally {
-        bindings.obx_bytes_array_free(bytesArray);
+        visitor.close();
       }
+      return results;
     });
   }
 
   /// Returns a list of [ids.length] Objects of type T, each corresponding to the location of its ID in [ids].
-  /// Non-existant IDs become null.
+  /// Non-existent IDs become null.
   List<T> getMany(List<int> ids) {
     if (ids.isEmpty) return [];
 
-    const bool allowMissing = true; // returns null if null is encountered in the data found
+    const bool allowMissing = true; // result includes null if an object is missing
     return OBX_id_array.executeWith(
         ids,
-        (ptr) => _getMany(allowMissing,
-            () => checkObxPtr(bindings.obx_box_get_many(_cBox, ptr), "failed to get many objects from box")));
+        (ptr) => _getMany(
+            allowMissing,
+            () => checkObxPtr(bindings.obx_box_get_many(_cBox, ptr), "failed to get many objects from box"),
+            (DataVisitor visitor) => checkObx(bindings.obx_box_visit_many(_cBox, ptr, visitor.fn, visitor.userData))));
   }
 
   /// Returns all stored objects in this Box.
   List<T> getAll() {
     const bool allowMissing = false; // throw if null is encountered in the data found
     return _getMany(
-        allowMissing, () => checkObxPtr(bindings.obx_box_get_all(_cBox), "failed to get all objects from box"));
+        allowMissing,
+        () => checkObxPtr(bindings.obx_box_get_all(_cBox), "failed to get all objects from box"),
+        (DataVisitor visitor) => checkObx(bindings.obx_box_visit_all(_cBox, visitor.fn, visitor.userData)));
   }
 
   /// Returns a builder to create queries for Object matching supplied criteria.

test/box_test.dart

@@ -78,7 +78,29 @@ void main() {
     }
   });
 
-  test(".getMany correctly handles non-existant items", () {
+  test(".getAll/getMany works on large arrays", () {
+    // This would fail on 32-bit system if objectbox-c obx_supports_bytes_array() wasn't respected
+    final length = 10 * 1000;
+    final largeString = 'A' * length;
+    expect(largeString.length, length);
+
+    box.put(TestEntity(tString: largeString));
+    box.put(TestEntity(tString: largeString));
+
+    List<TestEntity> items = box.getAll();
+    expect(items.length, 2);
+    expect(items[0].tString, largeString);
+    expect(items[1].tString, largeString);
+
+    box.put(TestEntity(tString: largeString));
+
+    items = box.getMany([1, 2]);
+    expect(items.length, 2);
+    expect(items[0].tString, largeString);
+    expect(items[1].tString, largeString);
+  });
+
+  test(".getMany correctly handles non-existent items", () {
     final List<TestEntity> items = ["One", "Two"].map((s) => TestEntity(tString: s)).toList();
     final List<int> ids = box.putMany(items);
     int otherId = 1;

test/query_test.dart

@@ -210,10 +210,11 @@ void main() {
     final largeString = 'A' * length;
     expect(largeString.length, length);
 
+    box.put(TestEntity(tString: largeString));
     box.put(TestEntity(tString: largeString));
     box.put(TestEntity(tString: largeString));
 
-    List<TestEntity> items = box.query(TestEntity_.id.greaterThan(0)).build().find();
+    List<TestEntity> items = box.query(TestEntity_.id.lessThan(3)).build().find();
     expect(items.length, 2);
     expect(items[0].tString, largeString);
     expect(items[1].tString, largeString);