diff --git a/lib/src/isolates/child.dart b/lib/src/isolates/child.dart index eba5cad..e08a358 100644 --- a/lib/src/isolates/child.dart +++ b/lib/src/isolates/child.dart @@ -1,9 +1,9 @@ import "dart:async"; import "dart:convert"; import "dart:io"; -import "dart:typed_data"; import "package:burt_network/burt_network.dart"; +import "package:dartcv4/dartcv.dart"; import "package:typed_isolate/typed_isolate.dart"; import "package:video/src/targeting/frame_properties.dart"; import "package:video/video.dart"; @@ -132,7 +132,7 @@ abstract class CameraIsolate extends IsolateChild final number = files.length; await File( "${cameraDirectory.path}/screenshot_$number.jpg", - ).writeAsBytes(jpegData); + ).writeAsBytes(jpegData.toU8List()); sendLog(Level.info, "Saved Screenshot"); sendToParent( FramePayload( @@ -202,17 +202,23 @@ abstract class CameraIsolate extends IsolateChild /// and get saved as a screenshot /// /// Most likely, this image will be too big to send over the network - Future getScreenshotJpeg(); + Future getScreenshotJpeg(); /// Sends an individual frame to the dashboard. /// /// This function also checks if the frame is too big to send, and if so, /// lowers the JPG quality by 1%. If the quality reaches 25% (visually noticeable), /// an error is logged instead. - void sendFrame(Uint8List image, {CameraDetails? detailsOverride}) { + void sendFrame(VecUChar image, {CameraDetails? detailsOverride}) { final details = detailsOverride ?? this.details; - if (image.length < maxPacketLength) { // Frame can be sent - sendToParent(FramePayload(details: details, image: image)); + // Frame can be sent + if (image.length < maxPacketLength) { + // Since we're sending the image's pointer address over an isolate, we don't + // want the image to be automatically released from memory since it will cause + // a segfault, we detach it from the finalizer and will release the memory manually + // from the parent isolate + VecUChar.finalizer.detach(image); + sendToParent(FramePayload(details: details, address: image.ptr.address)); } else if (details.quality > 25) { // Frame too large, lower quality sendLog(LogLevel.debug, "Lowering quality for $name from ${details.quality}"); details.quality -= 1; // maybe next frame can send diff --git a/lib/src/isolates/opencv.dart b/lib/src/isolates/opencv.dart index b6b8bc8..6b71cf8 100644 --- a/lib/src/isolates/opencv.dart +++ b/lib/src/isolates/opencv.dart @@ -1,5 +1,3 @@ -import "dart:typed_data"; - import "package:dartcv4/dartcv.dart"; import "package:burt_network/burt_network.dart"; import "package:video/src/targeting/frame_properties.dart"; @@ -118,7 +116,7 @@ class OpenCVCameraIsolate extends CameraIsolate { updateDetails(CameraDetails(streamWidth: streamWidth, streamHeight: streamHeight)); } - Uint8List? frame; + VecUChar? frame; // don't resize unless if the stream is different from the capture if (streamWidth < matrix.width || streamHeight < matrix.height) { try { @@ -158,7 +156,7 @@ class OpenCVCameraIsolate extends CameraIsolate { } @override - Future getScreenshotJpeg() async { + Future getScreenshotJpeg() async { if (camera == null) { return null; } diff --git a/lib/src/isolates/parent.dart b/lib/src/isolates/parent.dart index 96a512b..073ff7c 100644 --- a/lib/src/isolates/parent.dart +++ b/lib/src/isolates/parent.dart @@ -70,8 +70,16 @@ class CameraManager extends Service { await _commands?.cancel(); await _vision?.cancel(); - await _data?.cancel(); + // Dispose the parent isolate and kill all children before canceling + // data subscription, just in case if one last native frame is received await parent.dispose(); + + // Wait just a little bit to ensure any remaining messages get sent + // otherwise, if a message contained native memory, it will never + // be disposed + await Future.delayed(const Duration(milliseconds: 50)); + + await _data?.cancel(); } /// Handles data coming from the child isolates. @@ -81,7 +89,9 @@ class CameraManager extends Service { /// - If a [LogPayload] comes, logs the message using [logger]. void onData(IsolatePayload data) { switch (data) { - case FramePayload(:final details, :final image, :final screenshotPath): + case FramePayload(:final details, :final screenshotPath): + final image = data.image?.toU8List(); + data.dispose(); if (data.details.name == findObjectsInCameraFeed) { // Feeds from this camera get sent to the vision program. // The vision program will detect objects and send metadata to Autonomy. diff --git a/lib/src/isolates/payload.dart b/lib/src/isolates/payload.dart index 29e2f78..39db960 100644 --- a/lib/src/isolates/payload.dart +++ b/lib/src/isolates/payload.dart @@ -1,7 +1,7 @@ import "dart:ffi"; -import "dart:typed_data"; import "package:burt_network/burt_network.dart"; +import "package:dartcv4/dartcv.dart"; import "package:video/video.dart"; /// A payload containing some data to report back to the parent isolate. @@ -20,14 +20,26 @@ class FramePayload extends IsolatePayload { /// The details of the camera this frame came from. final CameraDetails details; - /// The image to send. - final Uint8List? image; + /// The native address of the image in memory + final int? address; /// The path of the screenshot String? screenshotPath; - /// A const constructor. - FramePayload({required this.details, this.image, this.screenshotPath}); + /// Const constructor for [FramePayload. + FramePayload({required this.details, this.address, this.screenshotPath}); + + /// The native image being sent from the pointer address + /// + /// This pointer will not be automatically freed from memory, call dispose() + /// when this will no longer be accessed + VecUChar? get image => address != null + ? VecUChar.fromPointer(Pointer.fromAddress(address!), attach: false) + : null; + + /// Frees the native image from memory, after this is called, + /// [image] should no longer be accessed + void dispose() => image?.dispose(); } /// A class to send log messages across isolates. The parent isolate is responsible for logging. diff --git a/lib/src/isolates/realsense.dart b/lib/src/isolates/realsense.dart index bd2eba9..82b3e96 100644 --- a/lib/src/isolates/realsense.dart +++ b/lib/src/isolates/realsense.dart @@ -1,5 +1,4 @@ import "dart:ffi"; -import "dart:typed_data"; import "package:burt_network/burt_network.dart"; import "package:dartcv4/dartcv.dart"; @@ -112,7 +111,7 @@ class RealSenseIsolate extends CameraIsolate { } @override - Future getScreenshotJpeg() async { + Future getScreenshotJpeg() async { // Get frames from RealSense final frames = camera.getFrames(); if (frames == nullptr) return null; @@ -171,7 +170,7 @@ class RealSenseIsolate extends CameraIsolate { updateDetails(CameraDetails(streamWidth: streamWidth, streamHeight: streamHeight)); } - Uint8List? frame; + VecUChar? frame; if (streamWidth < rgbMatrix.width || streamHeight < rgbMatrix.height) { try { // No idea why fx and fy are needed, but if they aren't present then diff --git a/lib/src/utils/opencv.dart b/lib/src/utils/opencv.dart index bf8669f..013e9eb 100644 --- a/lib/src/utils/opencv.dart +++ b/lib/src/utils/opencv.dart @@ -1,5 +1,4 @@ import "dart:ffi"; -import "dart:typed_data"; import "package:dartcv4/dartcv.dart"; import "package:video/realsense.dart"; @@ -61,9 +60,9 @@ extension MatrixUtils on Mat { static final _crosshairColor = Scalar.fromRgb(0, 255, 0); /// Encodes this image as a JPG with the given quality. - Uint8List? encodeJpg({required int quality}) { + VecUChar? encodeJpg({required int quality}) { final params = VecI32.fromList([IMWRITE_JPEG_QUALITY, quality]); - final (success, frame) = imencode(".jpg", this, params: params); + final (success, frame) = imencodeVec(".jpg", this, params: params); return success ? frame : null; } diff --git a/pubspec.yaml b/pubspec.yaml index aa5f9d8..febfb90 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -13,7 +13,7 @@ dependencies: typed_isolate: ^6.0.0 ffi: ^2.1.0 protobuf: ^3.1.0 - dartcv4: ^1.1.2 + dartcv4: ^1.1.4 dev_dependencies: ffigen: ^19.1.0