[camera_android_camerax] Fix NV21 Format

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.23
+
+* Converts NV21-compatible streamed images to NV21 when requested. In doing so,
+this plugin should now be compatible with [google_ml_kit_flutter](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master).
+
 ## 0.6.22
 
 * Implements `setDescriptionWhileRecording`.

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt

@@ -597,6 +597,12 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary
    */
   abstract fun getPigeonApiImageProxy(): PigeonApiImageProxy
 
+  /**
+   * An implementation of [PigeonApiImageProxyUtils] used to add a new Dart instance of
+   * `ImageProxyUtils` to the Dart `InstanceManager`.
+   */
+  abstract fun getPigeonApiImageProxyUtils(): PigeonApiImageProxyUtils
+
   /**
    * An implementation of [PigeonApiPlaneProxy] used to add a new Dart instance of `PlaneProxy` to
    * the Dart `InstanceManager`.
@@ -738,6 +744,7 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary
     PigeonApiAnalyzer.setUpMessageHandlers(binaryMessenger, getPigeonApiAnalyzer())
     PigeonApiLiveData.setUpMessageHandlers(binaryMessenger, getPigeonApiLiveData())
     PigeonApiImageProxy.setUpMessageHandlers(binaryMessenger, getPigeonApiImageProxy())
+    PigeonApiImageProxyUtils.setUpMessageHandlers(binaryMessenger, getPigeonApiImageProxyUtils())
     PigeonApiQualitySelector.setUpMessageHandlers(binaryMessenger, getPigeonApiQualitySelector())
     PigeonApiFallbackStrategy.setUpMessageHandlers(binaryMessenger, getPigeonApiFallbackStrategy())
     PigeonApiCameraControl.setUpMessageHandlers(binaryMessenger, getPigeonApiCameraControl())
@@ -785,6 +792,7 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary
     PigeonApiAnalyzer.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiLiveData.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiImageProxy.setUpMessageHandlers(binaryMessenger, null)
+    PigeonApiImageProxyUtils.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiQualitySelector.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiFallbackStrategy.setUpMessageHandlers(binaryMessenger, null)
     PigeonApiCameraControl.setUpMessageHandlers(binaryMessenger, null)
@@ -916,6 +924,8 @@ private class CameraXLibraryPigeonProxyApiBaseCodec(
       registrar.getPigeonApiLiveData().pigeon_newInstance(value) {}
     } else if (value is androidx.camera.core.ImageProxy) {
       registrar.getPigeonApiImageProxy().pigeon_newInstance(value) {}
+    } else if (value is ImageProxyUtils) {
+      registrar.getPigeonApiImageProxyUtils().pigeon_newInstance(value) {}
     } else if (value is androidx.camera.core.ImageProxy.PlaneProxy) {
       registrar.getPigeonApiPlaneProxy().pigeon_newInstance(value) {}
     } else if (value is androidx.camera.video.QualitySelector) {
@@ -5372,6 +5382,83 @@ abstract class PigeonApiImageProxy(
     }
   }
 }
+/** Utils for working with [ImageProxy]s. */
+@Suppress("UNCHECKED_CAST")
+abstract class PigeonApiImageProxyUtils(
+    open val pigeonRegistrar: CameraXLibraryPigeonProxyApiRegistrar
+) {
+  /**
+   * Returns a single Byte Buffer that is representative of the [planes] that are NV21 compatible.
+   */
+  abstract fun getNv21Buffer(
+      imageWidth: Long,
+      imageHeight: Long,
+      planes: List<androidx.camera.core.ImageProxy.PlaneProxy>
+  ): ByteArray
+
+  companion object {
+    @Suppress("LocalVariableName")
+    fun setUpMessageHandlers(binaryMessenger: BinaryMessenger, api: PigeonApiImageProxyUtils?) {
+      val codec = api?.pigeonRegistrar?.codec ?: CameraXLibraryPigeonCodec()
+      run {
+        val channel =
+            BasicMessageChannel<Any?>(
+                binaryMessenger,
+                "dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.getNv21Buffer",
+                codec)
+        if (api != null) {
+          channel.setMessageHandler { message, reply ->
+            val args = message as List<Any?>
+            val imageWidthArg = args[0] as Long
+            val imageHeightArg = args[1] as Long
+            val planesArg = args[2] as List<androidx.camera.core.ImageProxy.PlaneProxy>
+            val wrapped: List<Any?> =
+                try {
+                  listOf(api.getNv21Buffer(imageWidthArg, imageHeightArg, planesArg))
+                } catch (exception: Throwable) {
+                  CameraXLibraryPigeonUtils.wrapError(exception)
+                }
+            reply.reply(wrapped)
+          }
+        } else {
+          channel.setMessageHandler(null)
+        }
+      }
+    }
+  }
+
+  @Suppress("LocalVariableName", "FunctionName")
+  /** Creates a Dart instance of ImageProxyUtils and attaches it to [pigeon_instanceArg]. */
+  fun pigeon_newInstance(pigeon_instanceArg: ImageProxyUtils, callback: (Result<Unit>) -> Unit) {
+    if (pigeonRegistrar.ignoreCallsToDart) {
+      callback(
+          Result.failure(
+              CameraXError("ignore-calls-error", "Calls to Dart are being ignored.", "")))
+    } else if (pigeonRegistrar.instanceManager.containsInstance(pigeon_instanceArg)) {
+      callback(Result.success(Unit))
+    } else {
+      val pigeon_identifierArg =
+          pigeonRegistrar.instanceManager.addHostCreatedInstance(pigeon_instanceArg)
+      val binaryMessenger = pigeonRegistrar.binaryMessenger
+      val codec = pigeonRegistrar.codec
+      val channelName =
+          "dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.pigeon_newInstance"
+      val channel = BasicMessageChannel<Any?>(binaryMessenger, channelName, codec)
+      channel.send(listOf(pigeon_identifierArg)) {
+        if (it is List<*>) {
+          if (it.size > 1) {
+            callback(
+                Result.failure(CameraXError(it[0] as String, it[1] as String, it[2] as String?)))
+          } else {
+            callback(Result.success(Unit))
+          }
+        } else {
+          callback(Result.failure(CameraXLibraryPigeonUtils.createConnectionError(channelName)))
+        }
+      }
+    }
+  }
+}
 /**
  * A plane proxy which has an analogous interface as `android.media.Image.Plane`.
  *

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtils.java

@@ -0,0 +1,72 @@
+// Copyright 2013 The Flutter Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/* Utilities for working with {@code ImageProxy}s. */
+public class ImageProxyUtils {
+
+  /**
+   * Converts list of {@link PlaneProxy}s in YUV_420_888 format (with VU planes in NV21 layout) to a
+   * single NV21 {@code ByteBuffer}.
+   */
+  @NonNull
+  public static ByteBuffer planesToNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    if (!areUVPlanesNV21(planes, width, height)) {
+      throw new IllegalArgumentException(
+          "Provided UV planes are not in NV21 layout and thus cannot be converted.");
+    }
+
+    int imageSize = width * height;
+    int nv21Size = imageSize + 2 * (imageSize / 4);
+    byte[] nv21Bytes = new byte[nv21Size];
+
+    // Copy Y plane.
+    ByteBuffer yBuffer = planes.get(0).getBuffer();
+    yBuffer.rewind();
+    yBuffer.get(nv21Bytes, 0, imageSize);
+
+    // Copy interleaved VU plane (NV21 layout).
+    ByteBuffer vBuffer = planes.get(2).getBuffer();
+    ByteBuffer uBuffer = planes.get(1).getBuffer();
+
+    vBuffer.rewind();
+    uBuffer.rewind();
+    vBuffer.get(nv21Bytes, imageSize, 1);
+    uBuffer.get(nv21Bytes, imageSize + 1, 2 * imageSize / 4 - 1);
+
+    return ByteBuffer.wrap(nv21Bytes);
+  }
+
+  public static boolean areUVPlanesNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    int imageSize = width * height;
+
+    ByteBuffer uBuffer = planes.get(1).getBuffer();
+    ByteBuffer vBuffer = planes.get(2).getBuffer();
+
+    // Backup buffer properties.
+    int vBufferPosition = vBuffer.position();
+    int uBufferLimit = uBuffer.limit();
+
+    // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value.
+    vBuffer.position(vBufferPosition + 1);
+    // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value.
+    uBuffer.limit(uBufferLimit - 1);
+
+    // Check that the buffers are equal and have the expected number of elements.
+    boolean areNV21 =
+        (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0);
+
+    // Restore buffers to their initial state.
+    vBuffer.position(vBufferPosition);
+    uBuffer.limit(uBufferLimit);
+
+    return areNV21;
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtilsProxyApi.java

@@ -0,0 +1,37 @@
+// Copyright 2013 The Flutter Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/**
+ * ProxyApi implementation for {@link ImageProxyUtils}. This class may handle instantiating native
+ * object instances that are attached to a Dart instance or handle method calls on the associated
+ * native class or an instance of that class.
+ */
+public class ImageProxyUtilsProxyApi extends PigeonApiImageProxyUtils {
+  ImageProxyUtilsProxyApi(@NonNull ProxyApiRegistrar pigeonRegistrar) {
+    super(pigeonRegistrar);
+  }
+
+  // List<? extends PlaneProxy> can be considered the same as List<PlaneProxy>.
+  @SuppressWarnings("unchecked")
+  @NonNull
+  @Override
+  public byte[] getNv21Buffer(
+      long imageWidth, long imageHeight, @NonNull List<? extends PlaneProxy> planes) {
+    final ByteBuffer nv21Buffer =
+        ImageProxyUtils.planesToNV21(
+            (List<PlaneProxy>) planes, (int) imageWidth, (int) imageHeight);
+
+    byte[] bytes = new byte[nv21Buffer.remaining()];
+    nv21Buffer.get(bytes, 0, bytes.length);
+
+    return bytes;
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java

@@ -425,4 +425,10 @@ public PigeonApiMeteringPointFactory getPigeonApiMeteringPointFactory() {
   public CameraPermissionsErrorProxyApi getPigeonApiCameraPermissionsError() {
     return new CameraPermissionsErrorProxyApi(this);
   }
+
+  @NonNull
+  @Override
+  public PigeonApiImageProxyUtils getPigeonApiImageProxyUtils() {
+    return new ImageProxyUtilsProxyApi(this);
+  }
 }

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsApiTest.java

@@ -0,0 +1,48 @@
+// Copyright 2013 The Flutter Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.mockito.Mockito.mockStatic;
+
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+public class ImageProxyUtilsApiTest {
+
+  @Test
+  public void getNv21Buffer_returnsExpectedBytes() {
+    final PigeonApiImageProxyUtils api = new TestProxyApiRegistrar().getPigeonApiImageProxyUtils();
+
+    List<PlaneProxy> planes =
+        Arrays.asList(
+            Mockito.mock(PlaneProxy.class),
+            Mockito.mock(PlaneProxy.class),
+            Mockito.mock(PlaneProxy.class));
+    long width = 4;
+    long height = 2;
+    byte[] expectedBytes = new byte[] {1, 2, 3, 4, 5};
+    ByteBuffer mockBuffer = ByteBuffer.wrap(expectedBytes);
+
+    try (MockedStatic<ImageProxyUtils> mockedStatic = mockStatic(ImageProxyUtils.class)) {
+      mockedStatic
+          .when(
+              () ->
+                  ImageProxyUtils.planesToNV21(
+                      Mockito.anyList(), Mockito.anyInt(), Mockito.anyInt()))
+          .thenReturn(mockBuffer);
+
+      byte[] result = api.getNv21Buffer(width, height, planes);
+
+      assertArrayEquals(expectedBytes, result);
+      mockedStatic.verify(() -> ImageProxyUtils.planesToNV21(planes, (int) width, (int) height));
+    }
+  }
+}