From ca6518f0b79e36bcda7d311bf93a723cb85dbacf Mon Sep 17 00:00:00 2001
From: Marc Rousavy <me@mrousavy.com>
Date: Tue, 19 Mar 2024 11:18:43 +0100
Subject: [PATCH] feat: Basic read-only Frame Processors (`ImageAnalysis`
 Use-Case) (#2664)

* feat: Basic read-only Frame Processors (`ImageAnalysis` Use-Case)

* feat: Set target resolution

* Update CameraSession.kt

* Refactor

* fix: Fix Frame Processor not being unbound
---
 .../java/com/mrousavy/camera/CameraView.kt    |  18 +-
 .../camera/core/CameraConfiguration.kt        |   5 +-
 .../com/mrousavy/camera/core/CameraSession.kt |  53 ++--
 .../camera/core/FrameProcessorEffect.kt       | 244 ------------------
 .../camera/core/FrameProcessorPipeline.kt     |  24 ++
 5 files changed, 64 insertions(+), 280 deletions(-)
 delete mode 100644 package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorEffect.kt
 create mode 100644 package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorPipeline.kt

diff --git a/package/android/src/main/java/com/mrousavy/camera/CameraView.kt b/package/android/src/main/java/com/mrousavy/camera/CameraView.kt
index 86cdcead0c..df74724e5a 100644
--- a/package/android/src/main/java/com/mrousavy/camera/CameraView.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/CameraView.kt
@@ -173,20 +173,20 @@ class CameraView(context: Context) :
           config.photo = CameraConfiguration.Output.Disabled.create()
         }
 
-        // Video/Frame Processor
+        // Video
         if (video || enableFrameProcessor) {
-          config.video = CameraConfiguration.Output.Enabled.create(
-            CameraConfiguration.Video(
-              videoHdr,
-              pixelFormat,
-              enableFrameProcessor,
-              enableGpuBuffers
-            )
-          )
+          config.video = CameraConfiguration.Output.Enabled.create(CameraConfiguration.Video(videoHdr, pixelFormat))
         } else {
           config.video = CameraConfiguration.Output.Disabled.create()
         }
 
+        // Frame Processor
+        if (enableFrameProcessor) {
+          config.frameProcessor = CameraConfiguration.Output.Enabled.create(CameraConfiguration.FrameProcessor(Unit))
+        } else {
+          config.frameProcessor = CameraConfiguration.Output.Disabled.create()
+        }
+
         // Audio
         if (audio) {
           config.audio = CameraConfiguration.Output.Enabled.create(CameraConfiguration.Audio(Unit))
diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt
index 9ab78297c2..48da93faaf 100644
--- a/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt
@@ -17,6 +17,7 @@ data class CameraConfiguration(
   var preview: Output<Preview> = Output.Disabled.create(),
   var photo: Output<Photo> = Output.Disabled.create(),
   var video: Output<Video> = Output.Disabled.create(),
+  var frameProcessor: Output<FrameProcessor> = Output.Disabled.create(),
   var codeScanner: Output<CodeScanner> = Output.Disabled.create(),
 
   // Orientation
@@ -44,7 +45,8 @@ data class CameraConfiguration(
   // Output<T> types, those need to be comparable
   data class CodeScanner(val codeTypes: List<CodeType>)
   data class Photo(val enableHdr: Boolean, val photoQualityBalance: QualityBalance)
-  data class Video(val enableHdr: Boolean, val pixelFormat: PixelFormat, val enableFrameProcessor: Boolean, val enableGpuBuffers: Boolean)
+  data class Video(val enableHdr: Boolean, val pixelFormat: PixelFormat)
+  data class FrameProcessor(val nothing: Unit)
   data class Audio(val nothing: Unit)
   data class Preview(val surfaceProvider: SurfaceProvider)
 
@@ -92,6 +94,7 @@ data class CameraConfiguration(
         left.video != right.video ||
         left.enableLowLightBoost != right.enableLowLightBoost ||
         left.videoStabilizationMode != right.videoStabilizationMode ||
+        left.frameProcessor != right.frameProcessor ||
         left.codeScanner != right.codeScanner ||
         left.preview != right.preview ||
         left.format != right.format ||
diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
index de26257c7d..2b0ceb0b6f 100644
--- a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
@@ -15,6 +15,7 @@ import androidx.camera.core.CameraControl
 import androidx.camera.core.CameraSelector
 import androidx.camera.core.CameraState
 import androidx.camera.core.DynamicRange
+import androidx.camera.core.ExperimentalGetImage
 import androidx.camera.core.FocusMeteringAction
 import androidx.camera.core.ImageAnalysis
 import androidx.camera.core.ImageCapture
@@ -22,7 +23,7 @@ import androidx.camera.core.MeteringPoint
 import androidx.camera.core.MirrorMode
 import androidx.camera.core.Preview
 import androidx.camera.core.TorchState
-import androidx.camera.core.UseCaseGroup
+import androidx.camera.core.UseCase
 import androidx.camera.core.resolutionselector.ResolutionSelector
 import androidx.camera.extensions.ExtensionMode
 import androidx.camera.lifecycle.ProcessCameraProvider
@@ -80,11 +81,13 @@ class CameraSession(private val context: Context, private val callback: Callback
   private var previewOutput: Preview? = null
   private var photoOutput: ImageCapture? = null
   private var videoOutput: VideoCapture<Recorder>? = null
+  private var frameProcessorOutput: ImageAnalysis? = null
   private var codeScannerOutput: ImageAnalysis? = null
+  private val useCases: List<UseCase>
+    get() = listOfNotNull(previewOutput, photoOutput, videoOutput, frameProcessorOutput, codeScannerOutput)
 
   // Camera Outputs State
   private var recorderOutput: Recorder? = null
-  private var frameProcessorEffect: FrameProcessorEffect? = null
 
   // Camera State
   private val mutex = Mutex()
@@ -212,6 +215,7 @@ class CameraSession(private val context: Context, private val callback: Callback
     }
   }
 
+  @OptIn(ExperimentalGetImage::class)
   @SuppressLint("RestrictedApi")
   @Suppress("LiftReturnOrAssignment")
   private fun configureOutputs(configuration: CameraConfiguration) {
@@ -221,8 +225,6 @@ class CameraSession(private val context: Context, private val callback: Callback
 
     Log.i(TAG, "Using FPS Range: $fpsRange")
 
-    // TODO: Check if all of the values we set are supported with Video/Photo/Preview Capabilities from CameraInfo.
-
     // 1. Preview
     val previewConfig = configuration.preview as? CameraConfiguration.Output.Enabled<CameraConfiguration.Preview>
     if (previewConfig != null) {
@@ -321,15 +323,27 @@ class CameraSession(private val context: Context, private val callback: Callback
       recorderOutput = null
     }
 
-    // 3.5 Frame Processor (middleman)
-    if (videoConfig != null && videoConfig.config.enableFrameProcessor) {
-      // The FrameProcessorEffect is a middle-man between the Camera stream and the output surfaces.
-      frameProcessorEffect = FrameProcessorEffect(videoConfig.config.pixelFormat, videoConfig.config.enableGpuBuffers, callback)
+    // 4. Frame Processor
+    val frameProcessorConfig = configuration.frameProcessor as? CameraConfiguration.Output.Enabled<CameraConfiguration.FrameProcessor>
+    if (frameProcessorConfig != null) {
+      Log.i(TAG, "Creating Frame Processor output...")
+      val analyzer = ImageAnalysis.Builder().also { analysis ->
+        analysis.setBackpressureStrategy(ImageAnalysis.STRATEGY_BLOCK_PRODUCER)
+        analysis.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888)
+        if (format != null) {
+          Log.i(TAG, "Frame Processor size: ${format.videoSize}")
+          val resolutionSelector = ResolutionSelector.Builder().forSize(format.videoSize)
+          analysis.setResolutionSelector(resolutionSelector.build())
+        }
+      }.build()
+      val pipeline = FrameProcessorPipeline(callback)
+      analyzer.setAnalyzer(CameraQueues.videoQueue.executor, pipeline)
+      frameProcessorOutput = analyzer
     } else {
-      frameProcessorEffect = null
+      frameProcessorOutput = null
     }
 
-    // 4. Code Scanner
+    // 5. Code Scanner
     val codeScannerConfig = configuration.codeScanner as? CameraConfiguration.Output.Enabled<CameraConfiguration.CodeScanner>
     if (codeScannerConfig != null) {
       Log.i(TAG, "Creating CodeScanner output...")
@@ -344,7 +358,6 @@ class CameraSession(private val context: Context, private val callback: Callback
   }
 
   private fun closeCurrentOutputs(provider: ProcessCameraProvider) {
-    val useCases = listOfNotNull(previewOutput, photoOutput, videoOutput, codeScannerOutput)
     if (useCases.isEmpty()) {
       return
     }
@@ -353,13 +366,11 @@ class CameraSession(private val context: Context, private val callback: Callback
   }
 
   @SuppressLint("RestrictedApi")
-  @Suppress("LiftReturnOrAssignment")
   private suspend fun configureCamera(provider: ProcessCameraProvider, configuration: CameraConfiguration) {
     Log.i(TAG, "Binding Camera #${configuration.cameraId}...")
     checkCameraPermission()
 
     // Outputs
-    val useCases = listOfNotNull(previewOutput, photoOutput, videoOutput, codeScannerOutput)
     if (useCases.isEmpty()) {
       throw NoOutputsError()
     }
@@ -394,19 +405,9 @@ class CameraSession(private val context: Context, private val callback: Callback
       cameraSelector = cameraSelector.withExtension(context, provider, needsImageAnalysis, ExtensionMode.NIGHT, "NIGHT")
     }
 
-    // Frame Processor is a CameraEffect (Surface middleman)
-    val frameProcessorEffect = frameProcessorEffect
-    if (frameProcessorEffect != null) {
-      val useCaseGroup = UseCaseGroup.Builder()
-      useCases.forEach { useCase -> useCaseGroup.addUseCase(useCase) }
-      useCaseGroup.addEffect(frameProcessorEffect)
-
-      // Bind it all together (must be on UI Thread)
-      camera = provider.bindToLifecycle(this, cameraSelector, useCaseGroup.build())
-    } else {
-      // Bind it all together (must be on UI Thread)
-      camera = provider.bindToLifecycle(this, cameraSelector, *useCases.toTypedArray())
-    }
+    // Bind it all together (must be on UI Thread)
+    Log.i(TAG, "Binding ${useCases.size} use-cases...")
+    camera = provider.bindToLifecycle(this, cameraSelector, *useCases.toTypedArray())
 
     // Listen to Camera events
     var lastState = CameraState.Type.OPENING
diff --git a/package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorEffect.kt b/package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorEffect.kt
deleted file mode 100644
index 753eb112d0..0000000000
--- a/package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorEffect.kt
+++ /dev/null
@@ -1,244 +0,0 @@
-package com.mrousavy.camera.core
-
-import android.annotation.SuppressLint
-import android.hardware.HardwareBuffer
-import android.media.ImageReader
-import android.media.ImageWriter
-import android.os.Build
-import android.util.Log
-import androidx.annotation.ChecksSdkIntAtLeast
-import androidx.annotation.GuardedBy
-import androidx.annotation.RequiresApi
-import androidx.camera.core.CameraEffect
-import androidx.camera.core.CameraSelector
-import androidx.camera.core.SurfaceOutput
-import androidx.camera.core.SurfaceProcessor
-import androidx.camera.core.SurfaceRequest
-import com.mrousavy.camera.frameprocessor.Frame
-import com.mrousavy.camera.types.Orientation
-import com.mrousavy.camera.types.PixelFormat
-import com.mrousavy.camera.utils.ImageFormatUtils
-
-@SuppressLint("RestrictedApi")
-class FrameProcessorEffect(
-  format: PixelFormat = PixelFormat.NATIVE,
-  enableGpuBuffers: Boolean = false,
-  callback: CameraSession.Callback,
-  targets: Int = PREVIEW,
-  processor: FrameProcessorSurfaceProcessor = FrameProcessorSurfaceProcessor(format, enableGpuBuffers, callback)
-) : CameraEffect(
-  targets,
-  CameraQueues.videoQueue.executor,
-  processor,
-  { error -> callback.onError(error) }
-) {
-
-  class FrameProcessorSurfaceProcessor(
-    private val format: PixelFormat,
-    private val enableGpuBuffers: Boolean,
-    private val callback: CameraSession.Callback
-  ) : SurfaceProcessor {
-    companion object {
-      private const val TAG = "FrameProcessorEffect"
-      private const val MAX_IMAGES = 3
-    }
-    private val queue = CameraQueues.videoQueue
-    private val lock = Any()
-
-    @GuardedBy("lock")
-    private var imageWriter: ImageWriter? = null
-
-    override fun onInputSurface(request: SurfaceRequest) {
-      val requestedSize = request.resolution
-      val requestedFormat = request.deferrableSurface.prescribedStreamFormat
-      Log.i(TAG, "Requested new input surface: $requestedSize in format ${ImageFormatUtils.imageFormatToString(requestedFormat)}")
-
-      // The actual format we use might be different than the format of the output Surface (e.g. SurfaceView/MediaRecorder),
-      // because the user might want YUV images while the output Surface is PRIVATE.
-      // Since Android Q, ImageWriters can convert between such formats, so if that is possible, we will use a custom format,
-      // otherwise we will need to fall back to the default format.
-      var actualFormat = format.toImageFormat()
-      if (!isImageWriterCustomFormatsSupported() && actualFormat != requestedFormat) {
-        Log.w(
-          TAG,
-          "Trying to use format ${ImageFormatUtils.imageFormatToString(actualFormat)}, but output " +
-            "surface is ${ImageFormatUtils.imageFormatToString(requestedFormat)} " +
-            "and ImageWriters with custom formats are not available. " +
-            "Falling back to using format ${ImageFormatUtils.imageFormatToString(requestedFormat)}..."
-        )
-        actualFormat = requestedFormat
-      }
-      Log.i(
-        TAG,
-        "Creating ImageReader (${ImageFormatUtils.imageFormatToString(actualFormat)}) -> " +
-          "ImageWriter (${ImageFormatUtils.imageFormatToString(requestedFormat)}) pipeline..."
-      )
-
-      val imageReader = if (enableGpuBuffers && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
-        // Use GPU buffer flags for ImageReader for faster forwarding
-        val flags = getRecommendedHardwareBufferFlags(requestedSize.width, requestedSize.height)
-        Log.i(TAG, "Creating ImageReader with new GPU-Buffers API... (Usage Flags: $flags)")
-        ImageReader.newInstance(requestedSize.width, requestedSize.height, actualFormat, MAX_IMAGES, flags)
-      } else {
-        // Use default CPU flags for ImageReader
-        Log.i(TAG, "Creating ImageReader with default CPU usage flag...")
-        ImageReader.newInstance(requestedSize.width, requestedSize.height, actualFormat, MAX_IMAGES)
-      }
-
-      val cameraInfo = request.camera.cameraInfo
-      val orientation = Orientation.fromRotationDegrees(cameraInfo.sensorRotationDegrees)
-      val isMirrored = cameraInfo.lensFacing == CameraSelector.LENS_FACING_FRONT
-
-      imageReader.setOnImageAvailableListener({ reader ->
-        synchronized(lock) {
-          try {
-            val image = reader.acquireLatestImage() ?: return@setOnImageAvailableListener
-
-            val frame = Frame(image, image.timestamp, orientation, isMirrored)
-
-            frame.incrementRefCount()
-            try {
-              callback.onFrame(frame)
-
-              val imageWriter = imageWriter
-              if (imageWriter != null) {
-                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
-                  imageWriter.queueInputImage(image)
-                }
-              }
-            } finally {
-              frame.decrementRefCount()
-            }
-          } catch (e: Throwable) {
-            Log.e(TAG, "Failed to process image! ${e.message}", e)
-            callback.onError(e)
-          }
-        }
-      }, CameraQueues.videoQueue.handler)
-
-      // Submit the Surface to CameraX which will start streaming
-      request.provideSurface(imageReader.surface, queue.executor) { result ->
-        onImageReaderSurfaceClosed(imageReader, result.resultCode)
-      }
-    }
-
-    override fun onOutputSurface(surfaceOutput: SurfaceOutput) {
-      val requestedFormat = surfaceOutput.format
-      Log.i(TAG, "Received new output surface: ${surfaceOutput.size} in format ${ImageFormatUtils.imageFormatToString(requestedFormat)}")
-
-      var imageWriter: ImageWriter? = null
-      val surface = surfaceOutput.getSurface(queue.executor) { event ->
-        onOutputSurfaceClosed(event, imageWriter)
-      }
-
-      if (isImageWriterCustomFormatsSupported()) {
-        // Use custom target format, ImageWriter might be able to convert between the formats.
-        val customFormat = format.toImageFormat()
-        Log.i(TAG, "Creating ImageWriter with target format ${ImageFormatUtils.imageFormatToString(customFormat)}...")
-        imageWriter = ImageWriter.newInstance(surface, MAX_IMAGES, customFormat)
-      } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
-        // Use default format, ImageWriter might not be able to convert between the formats and crash....
-        Log.i(TAG, "Creating ImageWriter with default format (${ImageFormatUtils.imageFormatToString(requestedFormat)})...")
-        imageWriter = ImageWriter.newInstance(surface, MAX_IMAGES)
-      } else {
-        // ImageWriters are not available at all.
-        val error = RecordingWhileFrameProcessingUnavailable()
-        Log.e(TAG, error.message)
-        callback.onError(error)
-      }
-
-      synchronized(lock) {
-        this.imageWriter = imageWriter
-      }
-    }
-
-    private fun onImageReaderSurfaceClosed(imageReader: ImageReader, resultCode: Int) {
-      synchronized(lock) {
-        when (resultCode) {
-          SurfaceRequest.Result.RESULT_SURFACE_USED_SUCCESSFULLY -> Log.i(TAG, "Camera is done using $imageReader!")
-          SurfaceRequest.Result.RESULT_INVALID_SURFACE -> Log.e(TAG, "Camera could not use $imageReader - invalid Surface!")
-          SurfaceRequest.Result.RESULT_SURFACE_ALREADY_PROVIDED -> Log.i(TAG, "Camera already used a different Surface!")
-          SurfaceRequest.Result.RESULT_REQUEST_CANCELLED -> Log.i(TAG, "Surface Request has been cancelled.!")
-          SurfaceRequest.Result.RESULT_WILL_NOT_PROVIDE_SURFACE -> Log.i(TAG, "Surface Request ignored.")
-          else -> throw Error("Invalid SurfaceRequest Result State!")
-        }
-        Log.i(TAG, "Closing ImageReader $imageReader...")
-        imageReader.close()
-      }
-    }
-
-    private fun onOutputSurfaceClosed(event: SurfaceOutput.Event, imageWriter: ImageWriter?) {
-      synchronized(lock) {
-        Log.i(TAG, "Output Surface has been closed! Code: ${event.eventCode}")
-
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
-          Log.i(TAG, "Closing ImageWriter $imageWriter...")
-          imageWriter?.close()
-          if (this.imageWriter == imageWriter) {
-            this.imageWriter = null
-          }
-        }
-
-        event.surfaceOutput.close()
-      }
-    }
-
-    @ChecksSdkIntAtLeast(api = Build.VERSION_CODES.Q)
-    private fun isImageWriterCustomFormatsSupported(): Boolean {
-      // Since Android Q (API 29) ImageWriters can also automatically convert between ImageFormats.
-      // For example: ImageReader (YUV_420_888) -> ImageWriter (PRIVATE) -> SurfaceView (PRIVATE)
-      return Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q
-    }
-
-    /**
-     * Get the recommended HardwareBuffer flags for creating ImageReader instances with.
-     *
-     * Tries to use [HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE] if possible, [HardwareBuffer.USAGE_CPU_READ_OFTEN]
-     * or a combination of both flags if CPU access is needed, and [0] otherwise.
-     */
-    @RequiresApi(Build.VERSION_CODES.Q)
-    @Suppress("LiftReturnOrAssignment")
-    private fun getRecommendedHardwareBufferFlags(width: Int, height: Int): Long {
-      val cpuFlag = HardwareBuffer.USAGE_CPU_READ_OFTEN
-      val gpuFlag = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
-      val bothFlags = gpuFlag or cpuFlag
-
-      if (format == PixelFormat.NATIVE) {
-        // We don't need CPU access, so we can use GPU optimized buffers
-        if (supportsHardwareBufferFlags(width, height, gpuFlag)) {
-          // We support GPU Buffers directly and
-          Log.i(TAG, "GPU HardwareBuffers are supported!")
-          return gpuFlag
-        } else {
-          // no flags are supported - fall back to default
-          return 0
-        }
-      } else {
-        // We are using YUV or RGB formats, so we need CPU access on the Frame
-        if (supportsHardwareBufferFlags(width, height, bothFlags)) {
-          // We support both CPU and GPU flags!
-          Log.i(TAG, "GPU + CPU HardwareBuffers are supported!")
-          return bothFlags
-        } else if (supportsHardwareBufferFlags(width, height, cpuFlag)) {
-          // We only support a CPU read flag, that's fine
-          Log.i(TAG, "CPU HardwareBuffers are supported!")
-          return cpuFlag
-        } else {
-          // no flags are supported - fall back to default
-          return 0
-        }
-      }
-    }
-
-    @Suppress("LiftReturnOrAssignment")
-    @RequiresApi(Build.VERSION_CODES.Q)
-    private fun supportsHardwareBufferFlags(width: Int, height: Int, flags: Long): Boolean {
-      val hardwareBufferFormat = format.toHardwareBufferFormat()
-      try {
-        return HardwareBuffer.isSupported(width, height, hardwareBufferFormat, 1, flags)
-      } catch (_: Throwable) {
-        return false
-      }
-    }
-  }
-}
diff --git a/package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorPipeline.kt
new file mode 100644
index 0000000000..e0a9a86b1b
--- /dev/null
+++ b/package/android/src/main/java/com/mrousavy/camera/core/FrameProcessorPipeline.kt
@@ -0,0 +1,24 @@
+package com.mrousavy.camera.core
+
+import androidx.annotation.OptIn
+import androidx.camera.core.ExperimentalGetImage
+import androidx.camera.core.ImageAnalysis.Analyzer
+import androidx.camera.core.ImageProxy
+import com.mrousavy.camera.frameprocessor.Frame
+import com.mrousavy.camera.types.Orientation
+
+class FrameProcessorPipeline(private val callback: CameraSession.Callback) : Analyzer {
+  @OptIn(ExperimentalGetImage::class)
+  override fun analyze(imageProxy: ImageProxy) {
+    val orientation = Orientation.fromRotationDegrees(imageProxy.imageInfo.rotationDegrees)
+    // TODO: Get isMirrored from transformation matrix?
+    val isMirrored = false
+    val frame = Frame(imageProxy.image, imageProxy.imageInfo.timestamp, orientation, isMirrored)
+    try {
+      frame.incrementRefCount()
+      callback.onFrame(frame)
+    } finally {
+      frame.decrementRefCount()
+    }
+  }
+}