From f845c33255802aa5092c1c64812a711ca94a8654 Mon Sep 17 00:00:00 2001 From: Gold87 <91761103+Gold872@users.noreply.github.com> Date: Tue, 3 Jun 2025 23:08:24 -0400 Subject: [PATCH 1/4] Send native pointer in FramePayload --- lib/src/isolates/child.dart | 14 ++++++++++---- lib/src/isolates/opencv.dart | 4 +--- lib/src/isolates/parent.dart | 8 ++++++-- lib/src/isolates/payload.dart | 22 +++++++++++++++++----- lib/src/isolates/realsense.dart | 3 +-- lib/src/utils/opencv.dart | 5 ++--- 6 files changed, 37 insertions(+), 19 deletions(-) diff --git a/lib/src/isolates/child.dart b/lib/src/isolates/child.dart index 07f6886..0735ee5 100644 --- a/lib/src/isolates/child.dart +++ b/lib/src/isolates/child.dart @@ -1,9 +1,9 @@ import "dart:async"; import "dart:convert"; import "dart:io"; -import "dart:typed_data"; import "package:burt_network/burt_network.dart"; +import "package:dartcv4/dartcv.dart"; import "package:typed_isolate/typed_isolate.dart"; import "package:video/src/targeting/frame_properties.dart"; import "package:video/video.dart"; @@ -147,10 +147,16 @@ abstract class CameraIsolate extends IsolateChild /// This function also checks if the frame is too big to send, and if so, /// lowers the JPG quality by 1%. If the quality reaches 25% (visually noticeable), /// an error is logged instead. - void sendFrame(Uint8List image, {CameraDetails? detailsOverride}) { + void sendFrame(VecUChar image, {CameraDetails? detailsOverride}) { final details = detailsOverride ?? this.details; - if (image.length < maxPacketLength) { // Frame can be sent - sendToParent(FramePayload(details: details, image: image)); + // Frame can be sent + if (image.length < maxPacketLength) { + // Since we're sending the image's pointer address over an isolate, we don't + // want the image to be automatically released from memory since it will cause + // a segfault, we detach it from the finalizer and will release the memory manually + // from the parent isolate + VecUChar.finalizer.detach(image); + sendToParent(FramePayload(details: details, address: image.ptr.address)); } else if (details.quality > 25) { // Frame too large, lower quality sendLog(LogLevel.debug, "Lowering quality for $name from ${details.quality}"); details.quality -= 1; // maybe next frame can send diff --git a/lib/src/isolates/opencv.dart b/lib/src/isolates/opencv.dart index 17ad339..fd734ea 100644 --- a/lib/src/isolates/opencv.dart +++ b/lib/src/isolates/opencv.dart @@ -1,5 +1,3 @@ -import "dart:typed_data"; - import "package:dartcv4/dartcv.dart"; import "package:burt_network/burt_network.dart"; import "package:video/src/targeting/frame_properties.dart"; @@ -112,7 +110,7 @@ class OpenCVCameraIsolate extends CameraIsolate { updateDetails(CameraDetails(streamWidth: streamWidth, streamHeight: streamHeight)); } - Uint8List? frame; + VecUChar? frame; // don't resize unless if the stream is different from the capture if (streamWidth < matrix.width || streamHeight < matrix.height) { try { diff --git a/lib/src/isolates/parent.dart b/lib/src/isolates/parent.dart index 840565a..2a6e59e 100644 --- a/lib/src/isolates/parent.dart +++ b/lib/src/isolates/parent.dart @@ -70,8 +70,10 @@ class CameraManager extends Service { await _commands?.cancel(); await _vision?.cancel(); - await _data?.cancel(); + // Dispose the parent isolate and kill all children before canceling + // data subscription, just in case if one last native frame is received await parent.dispose(); + await _data?.cancel(); } /// Handles data coming from the child isolates. @@ -81,7 +83,9 @@ class CameraManager extends Service { /// - If a [LogPayload] comes, logs the message using [logger]. void onData(IsolatePayload data) { switch (data) { - case FramePayload(:final image, :final details): + case FramePayload(:final details): + final image = data.image?.toU8List(); + data.dispose(); if (data.details.name == findObjectsInCameraFeed) { // Feeds from this camera get sent to the vision program. // The vision program will detect objects and send metadata to Autonomy. diff --git a/lib/src/isolates/payload.dart b/lib/src/isolates/payload.dart index 175aa56..66b83f5 100644 --- a/lib/src/isolates/payload.dart +++ b/lib/src/isolates/payload.dart @@ -1,7 +1,7 @@ import "dart:ffi"; -import "dart:typed_data"; import "package:burt_network/burt_network.dart"; +import "package:dartcv4/dartcv.dart"; import "package:video/video.dart"; /// A payload containing some data to report back to the parent isolate. @@ -20,11 +20,23 @@ class FramePayload extends IsolatePayload { /// The details of the camera this frame came from. final CameraDetails details; - /// The image to send. - final Uint8List? image; + /// The native address of the image in memory + final int? address; - /// A const constructor. - FramePayload({required this.details, this.image}); + /// Const constructor for [FramePayload. + FramePayload({required this.details, this.address}); + + /// The native image being sent from the pointer address + /// + /// This pointer will not be automatically freed from memory, call dispose() + /// when this will no longer be accessed + VecUChar? get image => address != null + ? VecUChar.fromPointer(Pointer.fromAddress(address!), false) + : null; + + /// Frees the native image from memory, after this is called, + /// [image] should no longer be accessed + void dispose() => image?.dispose(); } /// A class to send log messages across isolates. The parent isolate is responsible for logging. diff --git a/lib/src/isolates/realsense.dart b/lib/src/isolates/realsense.dart index 6fe196b..06143dc 100644 --- a/lib/src/isolates/realsense.dart +++ b/lib/src/isolates/realsense.dart @@ -1,5 +1,4 @@ import "dart:ffi"; -import "dart:typed_data"; import "package:burt_network/burt_network.dart"; import "package:dartcv4/dartcv.dart"; @@ -144,7 +143,7 @@ class RealSenseIsolate extends CameraIsolate { updateDetails(CameraDetails(streamWidth: streamWidth, streamHeight: streamHeight)); } - Uint8List? frame; + VecUChar? frame; if (streamWidth < rgbMatrix.width || streamHeight < rgbMatrix.height) { try { // No idea why fx and fy are needed, but if they aren't present then diff --git a/lib/src/utils/opencv.dart b/lib/src/utils/opencv.dart index 7f0764a..f28a943 100644 --- a/lib/src/utils/opencv.dart +++ b/lib/src/utils/opencv.dart @@ -1,5 +1,4 @@ import "dart:ffi"; -import "dart:typed_data"; import "package:dartcv4/dartcv.dart"; import "package:video/realsense.dart"; @@ -61,9 +60,9 @@ extension MatrixUtils on Mat { static final _crosshairColor = Scalar.fromRgb(0, 255, 0); /// Encodes this image as a JPG with the given quality. - Uint8List? encodeJpg({required int quality}) { + VecUChar? encodeJpg({required int quality}) { final params = VecI32.fromList([IMWRITE_JPEG_QUALITY, quality]); - final (success, frame) = imencode(".jpg", this, params: params); + final (success, frame) = imencodeVec(".jpg", this, params: params); return success ? frame : null; } From 2ab03c1ce33989045dda014412a1b92cc7abc53f Mon Sep 17 00:00:00 2001 From: Gold87 <91761103+Gold872@users.noreply.github.com> Date: Tue, 3 Jun 2025 23:36:16 -0400 Subject: [PATCH 2/4] Update dartcv4 --- lib/src/isolates/payload.dart | 2 +- pubspec.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/isolates/payload.dart b/lib/src/isolates/payload.dart index 66b83f5..9abe39c 100644 --- a/lib/src/isolates/payload.dart +++ b/lib/src/isolates/payload.dart @@ -31,7 +31,7 @@ class FramePayload extends IsolatePayload { /// This pointer will not be automatically freed from memory, call dispose() /// when this will no longer be accessed VecUChar? get image => address != null - ? VecUChar.fromPointer(Pointer.fromAddress(address!), false) + ? VecUChar.fromPointer(Pointer.fromAddress(address!), attach: false) : null; /// Frees the native image from memory, after this is called, diff --git a/pubspec.yaml b/pubspec.yaml index 9d9f611..25ebefe 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -13,7 +13,7 @@ dependencies: typed_isolate: ^6.0.0 ffi: ^2.1.0 protobuf: ^3.1.0 - dartcv4: ^1.1.2 + dartcv4: ^1.1.4 dev_dependencies: ffigen: ^16.0.0 From 60795043be65c4d63a68d4957f2298b7117032a9 Mon Sep 17 00:00:00 2001 From: Gold87 <91761103+Gold872@users.noreply.github.com> Date: Thu, 10 Jul 2025 20:18:52 -0400 Subject: [PATCH 3/4] Add delay between parent dispose and data subscription dispose --- lib/src/isolates/parent.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/src/isolates/parent.dart b/lib/src/isolates/parent.dart index 2a6e59e..c69cc31 100644 --- a/lib/src/isolates/parent.dart +++ b/lib/src/isolates/parent.dart @@ -73,6 +73,12 @@ class CameraManager extends Service { // Dispose the parent isolate and kill all children before canceling // data subscription, just in case if one last native frame is received await parent.dispose(); + + // Wait just a little bit to ensure any remaining messages get sent + // otherwise, if a message contained native memory, it will never + // be disposed + await Future.delayed(const Duration(milliseconds: 50)); + await _data?.cancel(); } From f8b28a9dc7a51733c65ef5385fe8706c1c624259 Mon Sep 17 00:00:00 2001 From: Gold87 <91761103+Gold872@users.noreply.github.com> Date: Wed, 13 Aug 2025 17:49:59 -0400 Subject: [PATCH 4/4] Return VecUChar from screenshot data --- lib/src/isolates/child.dart | 4 ++-- lib/src/isolates/opencv.dart | 2 +- lib/src/isolates/realsense.dart | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/src/isolates/child.dart b/lib/src/isolates/child.dart index 68bffba..e08a358 100644 --- a/lib/src/isolates/child.dart +++ b/lib/src/isolates/child.dart @@ -132,7 +132,7 @@ abstract class CameraIsolate extends IsolateChild final number = files.length; await File( "${cameraDirectory.path}/screenshot_$number.jpg", - ).writeAsBytes(jpegData); + ).writeAsBytes(jpegData.toU8List()); sendLog(Level.info, "Saved Screenshot"); sendToParent( FramePayload( @@ -202,7 +202,7 @@ abstract class CameraIsolate extends IsolateChild /// and get saved as a screenshot /// /// Most likely, this image will be too big to send over the network - Future getScreenshotJpeg(); + Future getScreenshotJpeg(); /// Sends an individual frame to the dashboard. /// diff --git a/lib/src/isolates/opencv.dart b/lib/src/isolates/opencv.dart index 4f809ef..6b71cf8 100644 --- a/lib/src/isolates/opencv.dart +++ b/lib/src/isolates/opencv.dart @@ -156,7 +156,7 @@ class OpenCVCameraIsolate extends CameraIsolate { } @override - Future getScreenshotJpeg() async { + Future getScreenshotJpeg() async { if (camera == null) { return null; } diff --git a/lib/src/isolates/realsense.dart b/lib/src/isolates/realsense.dart index 1ccc3f3..82b3e96 100644 --- a/lib/src/isolates/realsense.dart +++ b/lib/src/isolates/realsense.dart @@ -111,7 +111,7 @@ class RealSenseIsolate extends CameraIsolate { } @override - Future getScreenshotJpeg() async { + Future getScreenshotJpeg() async { // Get frames from RealSense final frames = camera.getFrames(); if (frames == nullptr) return null;