Send image data as a raw pointer

lib/src/isolates/child.dart

@@ -1,9 +1,9 @@
 import "dart:async";
 import "dart:convert";
 import "dart:io";
-import "dart:typed_data";
 
 import "package:burt_network/burt_network.dart";
+import "package:dartcv4/dartcv.dart";
 import "package:typed_isolate/typed_isolate.dart";
 import "package:video/src/targeting/frame_properties.dart";
 import "package:video/video.dart";
@@ -132,7 +132,7 @@ abstract class CameraIsolate extends IsolateChild<IsolatePayload, VideoCommand>
         final number = files.length;
         await File(
           "${cameraDirectory.path}/screenshot_$number.jpg",
-        ).writeAsBytes(jpegData);
+        ).writeAsBytes(jpegData.toU8List());
         sendLog(Level.info, "Saved Screenshot");
         sendToParent(
           FramePayload(
@@ -202,17 +202,23 @@ abstract class CameraIsolate extends IsolateChild<IsolatePayload, VideoCommand>
   /// and get saved as a screenshot
   /// 
   /// Most likely, this image will be too big to send over the network
-  Future<Uint8List?> getScreenshotJpeg();
+  Future<VecUChar?> getScreenshotJpeg();
 
   /// Sends an individual frame to the dashboard.
   ///
   /// This function also checks if the frame is too big to send, and if so,
   /// lowers the JPG quality by 1%. If the quality reaches 25% (visually noticeable),
   /// an error is logged instead.
-  void sendFrame(Uint8List image, {CameraDetails? detailsOverride}) {
+  void sendFrame(VecUChar image, {CameraDetails? detailsOverride}) {
     final details = detailsOverride ?? this.details;
-    if (image.length < maxPacketLength) {  // Frame can be sent
-      sendToParent(FramePayload(details: details, image: image));
+    // Frame can be sent
+    if (image.length < maxPacketLength) {
+      // Since we're sending the image's pointer address over an isolate, we don't
+      // want the image to be automatically released from memory since it will cause
+      // a segfault, we detach it from the finalizer and will release the memory manually
+      // from the parent isolate
+      VecUChar.finalizer.detach(image);
+      sendToParent(FramePayload(details: details, address: image.ptr.address));
     } else if (details.quality > 25) {  // Frame too large, lower quality
       sendLog(LogLevel.debug, "Lowering quality for $name from ${details.quality}");
       details.quality -= 1;  // maybe next frame can send

lib/src/isolates/opencv.dart

@@ -1,5 +1,3 @@
-import "dart:typed_data";
-
 import "package:dartcv4/dartcv.dart";
 import "package:burt_network/burt_network.dart";
 import "package:video/src/targeting/frame_properties.dart";
@@ -118,7 +116,7 @@ class OpenCVCameraIsolate extends CameraIsolate {
       updateDetails(CameraDetails(streamWidth: streamWidth, streamHeight: streamHeight));
     }
 
-    Uint8List? frame;
+    VecUChar? frame;
     // don't resize unless if the stream is different from the capture
     if (streamWidth < matrix.width || streamHeight < matrix.height) {
       try {
@@ -158,7 +156,7 @@ class OpenCVCameraIsolate extends CameraIsolate {
   }
 
   @override
-  Future<Uint8List?> getScreenshotJpeg() async {
+  Future<VecUChar?> getScreenshotJpeg() async {
     if (camera == null) {
       return null;
     }

lib/src/isolates/parent.dart

@@ -70,8 +70,16 @@ class CameraManager extends Service {
 
     await _commands?.cancel();
     await _vision?.cancel();
-    await _data?.cancel();
+    // Dispose the parent isolate and kill all children before canceling
+    // data subscription, just in case if one last native frame is received
     await parent.dispose();
+
+    // Wait just a little bit to ensure any remaining messages get sent
+    // otherwise, if a message contained native memory, it will never
+    // be disposed
+    await Future<void>.delayed(const Duration(milliseconds: 50));
+
+    await _data?.cancel();
   }
 
   /// Handles data coming from the child isolates.
@@ -81,7 +89,9 @@ class CameraManager extends Service {
   /// - If a [LogPayload] comes, logs the message using [logger].
   void onData(IsolatePayload data) {
     switch (data) {
-      case FramePayload(:final details, :final image, :final screenshotPath):
+      case FramePayload(:final details, :final screenshotPath):
+        final image = data.image?.toU8List();
+        data.dispose();
         if (data.details.name == findObjectsInCameraFeed) {
           // Feeds from this camera get sent to the vision program.
           // The vision program will detect objects and send metadata to Autonomy.

lib/src/isolates/payload.dart

@@ -1,7 +1,7 @@
 import "dart:ffi";
-import "dart:typed_data";
 
 import "package:burt_network/burt_network.dart";
+import "package:dartcv4/dartcv.dart";
 import "package:video/video.dart";
 
 /// A payload containing some data to report back to the parent isolate.
@@ -20,14 +20,26 @@ class FramePayload extends IsolatePayload {
   /// The details of the camera this frame came from.
   final CameraDetails details;
 
-  /// The image to send.
-  final Uint8List? image;
+  /// The native address of the image in memory
+  final int? address;
 
   /// The path of the screenshot
   String? screenshotPath;
 
-  /// A const constructor.
-  FramePayload({required this.details, this.image, this.screenshotPath});
+  /// Const constructor for [FramePayload.
+  FramePayload({required this.details, this.address, this.screenshotPath});
+
+  /// The native image being sent from the pointer address
+  ///
+  /// This pointer will not be automatically freed from memory, call dispose()
+  /// when this will no longer be accessed
+  VecUChar? get image => address != null
+      ? VecUChar.fromPointer(Pointer.fromAddress(address!), attach: false)
+      : null;
+
+  /// Frees the native image from memory, after this is called,
+  /// [image] should no longer be accessed
+  void dispose() => image?.dispose();
 }
 
 /// A class to send log messages across isolates. The parent isolate is responsible for logging.

lib/src/isolates/realsense.dart

@@ -1,5 +1,4 @@
 import "dart:ffi";
-import "dart:typed_data";
 
 import "package:burt_network/burt_network.dart";
 import "package:dartcv4/dartcv.dart";
@@ -112,7 +111,7 @@ class RealSenseIsolate extends CameraIsolate {
   }
 
   @override
-  Future<Uint8List?> getScreenshotJpeg() async {
+  Future<VecUChar?> getScreenshotJpeg() async {
     // Get frames from RealSense
     final frames = camera.getFrames();
     if (frames == nullptr) return null;
@@ -171,7 +170,7 @@ class RealSenseIsolate extends CameraIsolate {
       updateDetails(CameraDetails(streamWidth: streamWidth, streamHeight: streamHeight));
     }
 
-    Uint8List? frame;
+    VecUChar? frame;
     if (streamWidth < rgbMatrix.width || streamHeight < rgbMatrix.height) {
       try {
         // No idea why fx and fy are needed, but if they aren't present then

lib/src/utils/opencv.dart

@@ -1,5 +1,4 @@
 import "dart:ffi";
-import "dart:typed_data";
 
 import "package:dartcv4/dartcv.dart";
 import "package:video/realsense.dart";
@@ -61,9 +60,9 @@ extension MatrixUtils on Mat {
   static final _crosshairColor = Scalar.fromRgb(0, 255, 0);
 
   /// Encodes this image as a JPG with the given quality.
-  Uint8List? encodeJpg({required int quality}) {
+  VecUChar? encodeJpg({required int quality}) {
     final params = VecI32.fromList([IMWRITE_JPEG_QUALITY, quality]);
-    final (success, frame) = imencode(".jpg", this, params: params);
+    final (success, frame) = imencodeVec(".jpg", this, params: params);
     return success ? frame : null;
   }
 

pubspec.yaml

@@ -13,7 +13,7 @@ dependencies:
   typed_isolate: ^6.0.0
   ffi: ^2.1.0
   protobuf: ^3.1.0
-  dartcv4: ^1.1.2
+  dartcv4: ^1.1.4
 
 dev_dependencies:
   ffigen: ^19.1.0