diff --git a/packages/camera/camera_android/CHANGELOG.md b/packages/camera/camera_android/CHANGELOG.md index 3408b0f0872..271503d02f1 100644 --- a/packages/camera/camera_android/CHANGELOG.md +++ b/packages/camera/camera_android/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.10.10+4 + +* Fix flutter#166533 - prevent startImageStream OOM error when main thread paused. + ## 0.10.10+3 * Waits for the creation of the capture session when initializing the camera to avoid thread race conditions. diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 1a9cf18307f..a4ff20e885c 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -9,11 +9,14 @@ import android.media.ImageReader; import android.os.Handler; import android.os.Looper; +import android.util.Log; import android.view.Surface; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; +import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -22,6 +25,7 @@ // Wraps an ImageReader to allow for testing of the image handler. public class ImageStreamReader { + private static final String TAG = "ImageStreamReader"; /** * The image format we are going to send back to dart. Usually it's the same as streamImageFormat @@ -33,6 +37,16 @@ public class ImageStreamReader { private final ImageReader imageReader; private final ImageStreamReaderUtils imageStreamReaderUtils; + @VisibleForTesting(otherwise = VisibleForTesting.NONE) + @Nullable + public Handler handler; + + /** + * This hard reference is required so frames don't get randomly dropped before reaching the main + * looper. + */ + private Map latestImageBufferHardReference = null; + /** * Creates a new instance of the {@link ImageStreamReader}. * @@ -95,40 +109,69 @@ public void onImageAvailable( @NonNull Image image, @NonNull CameraCaptureProperties captureProps, @NonNull EventChannel.EventSink imageStreamSink) { - try { - Map imageBuffer = new HashMap<>(); + Map imageBuffer = new HashMap<>(); + imageBuffer.put("width", image.getWidth()); + imageBuffer.put("height", image.getHeight()); + try { // Get plane data ready if (dartImageFormat == ImageFormat.NV21) { imageBuffer.put("planes", parsePlanesForNv21(image)); } else { imageBuffer.put("planes", parsePlanesForYuvOrJpeg(image)); } - - imageBuffer.put("width", image.getWidth()); - imageBuffer.put("height", image.getHeight()); - imageBuffer.put("format", dartImageFormat); - imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); - imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); - Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); - imageBuffer.put( - "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); - - final Handler handler = new Handler(Looper.getMainLooper()); - handler.post(() -> imageStreamSink.success(imageBuffer)); - image.close(); - } catch (IllegalStateException e) { - // Handle "buffer is inaccessible" errors that can happen on some devices from ImageStreamReaderUtils.yuv420ThreePlanesToNV21() - final Handler handler = new Handler(Looper.getMainLooper()); + // Handle "buffer is inaccessible" errors that can happen on some devices from + // ImageStreamReaderUtils.yuv420ThreePlanesToNV21() + final Handler handler = + this.handler != null ? this.handler : new Handler(Looper.getMainLooper()); handler.post( () -> imageStreamSink.error( "IllegalStateException", "Caught IllegalStateException: " + e.getMessage(), null)); + } finally { image.close(); } + + imageBuffer.put("format", dartImageFormat); + imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); + imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); + Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); + imageBuffer.put( + "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); + + final Handler handler = + this.handler != null ? this.handler : new Handler(Looper.getMainLooper()); + + // Keep a hard reference to the latest frame, so it isn't dropped before it reaches the main + // looper + latestImageBufferHardReference = imageBuffer; + + boolean postResult = + handler.post( + new Runnable() { + @VisibleForTesting public WeakReference> weakImageBuffer; + + public Runnable withImageBuffer(Map imageBuffer) { + weakImageBuffer = new WeakReference<>(imageBuffer); + return this; + } + + @Override + public void run() { + final Map imageBuffer = weakImageBuffer.get(); + if (imageBuffer == null) { + // The memory was freed by the runtime, most likely due to a memory build-up + // while the main thread was lagging. Frames are silently dropped in this + // case. + Log.d(TAG, "Image buffer was dropped by garbage collector."); + return; + } + imageStreamSink.success(imageBuffer); + } + }.withImageBuffer(imageBuffer)); } /** diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java index 22e7aab464c..a762fa2dc34 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java @@ -9,15 +9,22 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.graphics.ImageFormat; import android.media.Image; import android.media.ImageReader; +import android.os.Handler; import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; +import java.lang.ref.WeakReference; +import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; @@ -61,40 +68,9 @@ public void onImageAvailable_parsesPlanesForNv21() { when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())) .thenReturn(mockBytes); - // The image format as streamed from the camera - int imageFormat = ImageFormat.YUV_420_888; - - // Mock YUV image - Image mockImage = mock(Image.class); - when(mockImage.getWidth()).thenReturn(1280); - when(mockImage.getHeight()).thenReturn(720); - when(mockImage.getFormat()).thenReturn(imageFormat); - - // Mock planes. YUV images have 3 planes (Y, U, V). - Image.Plane planeY = mock(Image.Plane.class); - Image.Plane planeU = mock(Image.Plane.class); - Image.Plane planeV = mock(Image.Plane.class); - - // Y plane is width*height - // Row stride is generally == width but when there is padding it will - // be larger. The numbers in this example are from a Vivo V2135 on 'high' - // setting (1280x720). - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); - when(planeY.getRowStride()).thenReturn(1536); - when(planeY.getPixelStride()).thenReturn(1); - - // U and V planes are always the same sizes/values. - // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 - when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); - when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); - when(planeU.getRowStride()).thenReturn(1536); - when(planeV.getRowStride()).thenReturn(1536); - when(planeU.getPixelStride()).thenReturn(2); - when(planeV.getPixelStride()).thenReturn(2); - - // Add planes to image - Image.Plane[] planes = {planeY, planeU, planeV}; - when(mockImage.getPlanes()).thenReturn(planes); + // Note: the code for getImage() was previously inlined, with uSize set to one less than + // getImage() calculates (see function implementation) + Image mockImage = ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888); CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); @@ -102,7 +78,8 @@ public void onImageAvailable_parsesPlanesForNv21() { // Make sure we processed the frame with parsePlanesForNv21 verify(mockImageStreamReaderUtils) - .yuv420ThreePlanesToNV21(planes, mockImage.getWidth(), mockImage.getHeight()); + .yuv420ThreePlanesToNV21( + mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight()); } /** If we are requesting YUV420, then we should send the 3-plane image as it is. */ @@ -120,40 +97,9 @@ public void onImageAvailable_parsesPlanesForYuv420() { when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())) .thenReturn(mockBytes); - // The image format as streamed from the camera - int imageFormat = ImageFormat.YUV_420_888; - - // Mock YUV image - Image mockImage = mock(Image.class); - when(mockImage.getWidth()).thenReturn(1280); - when(mockImage.getHeight()).thenReturn(720); - when(mockImage.getFormat()).thenReturn(imageFormat); - - // Mock planes. YUV images have 3 planes (Y, U, V). - Image.Plane planeY = mock(Image.Plane.class); - Image.Plane planeU = mock(Image.Plane.class); - Image.Plane planeV = mock(Image.Plane.class); - - // Y plane is width*height - // Row stride is generally == width but when there is padding it will - // be larger. The numbers in this example are from a Vivo V2135 on 'high' - // setting (1280x720). - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); - when(planeY.getRowStride()).thenReturn(1536); - when(planeY.getPixelStride()).thenReturn(1); - - // U and V planes are always the same sizes/values. - // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 - when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); - when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); - when(planeU.getRowStride()).thenReturn(1536); - when(planeV.getRowStride()).thenReturn(1536); - when(planeU.getPixelStride()).thenReturn(2); - when(planeV.getPixelStride()).thenReturn(2); - - // Add planes to image - Image.Plane[] planes = {planeY, planeU, planeV}; - when(mockImage.getPlanes()).thenReturn(planes); + // Note: the code for getImage() was previously inlined, with uSize set to one less than + // getImage() calculates (see function implementation) + Image mockImage = ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888); CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); @@ -162,4 +108,72 @@ public void onImageAvailable_parsesPlanesForYuv420() { // Make sure we processed the frame with parsePlanesForYuvOrJpeg verify(mockImageStreamReaderUtils, never()).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()); } + + @Test + public void onImageAvailable_dropFramesWhenHandlerHalted() { + int dartImageFormat = ImageFormat.YUV_420_888; + + ImageReader mockImageReader = mock(ImageReader.class); + ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); + ImageStreamReader imageStreamReader = + new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils); + + for (boolean invalidateWeakReference : new boolean[] {true, false}) { + final List runnables = new ArrayList(); + + Handler mockHandler = mock(Handler.class); + imageStreamReader.handler = mockHandler; + + // initially, handler will simulate a hanging main looper, that only queues inputs + when(mockHandler.post(any(Runnable.class))) + .thenAnswer( + inputs -> { + Runnable r = inputs.getArgument(0, Runnable.class); + runnables.add(r); + return true; + }); + + CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); + EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); + + Image mockImage = + ImageStreamReaderTestUtils.getImage(1280, 720, 256, ImageFormat.YUV_420_888); + imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink); + + // make sure the image was closed, even when skipping frames + verify(mockImage, times(1)).close(); + + // check that we collected all runnables in this method + assertEquals(runnables.size(), 1); + + // verify post() was not called more times than it should have + verify(mockHandler, times(1)).post(any(Runnable.class)); + + // make sure callback was not yet invoked + verify(mockEventSink, never()).success(any(Map.class)); + + // simulate frame processing + for (Runnable r : runnables) { + if (invalidateWeakReference) { + // Replace the captured WeakReference with one pointing to null. + Field[] fields = r.getClass().getDeclaredFields(); + for (Field field : fields) { + if (field.getType().equals(WeakReference.class)) { + // Remove the `final` modifier + try { + field.set(r, new WeakReference>(null)); + } catch (IllegalAccessException e) { + throw new RuntimeException("Failed to inject null WeakReference", e); + } + } + } + } + + r.run(); + } + + // make sure all callbacks were invoked so far + verify(mockEventSink, invalidateWeakReference ? never() : times(1)).success(any(Map.class)); + } + } } diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTestUtils.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTestUtils.java new file mode 100644 index 00000000000..81e4815e653 --- /dev/null +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTestUtils.java @@ -0,0 +1,59 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camera.media; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import android.media.Image; +import java.nio.ByteBuffer; + +public class ImageStreamReaderTestUtils { + /** + * Creates a mock {@link android.media.Image} object for use in tests, simulating the specified + * dimensions, padding, and image format. + */ + public static Image getImage(int imageWidth, int imageHeight, int padding, int imageFormat) { + int rowStride = imageWidth + padding; + + int ySize = (rowStride * imageHeight) - padding; + int uSize = (ySize / 2) - (padding / 2); + int vSize = uSize; + + // Mock YUV image + Image mockImage = mock(Image.class); + when(mockImage.getWidth()).thenReturn(imageWidth); + when(mockImage.getHeight()).thenReturn(imageHeight); + when(mockImage.getFormat()).thenReturn(imageFormat); + + // Mock planes. YUV images have 3 planes (Y, U, V). + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + + // Y plane is width*height + // Row stride is generally == width but when there is padding it will + // be larger. + // Here we are adding 256 padding. + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(ySize)); + when(planeY.getRowStride()).thenReturn(rowStride); + when(planeY.getPixelStride()).thenReturn(1); + + // U and V planes are always the same sizes/values. + // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 + when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(uSize)); + when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(vSize)); + when(planeU.getRowStride()).thenReturn(rowStride); + when(planeV.getRowStride()).thenReturn(rowStride); + when(planeU.getPixelStride()).thenReturn(2); + when(planeV.getPixelStride()).thenReturn(2); + + // Add planes to image + Image.Plane[] planes = {planeY, planeU, planeV}; + when(mockImage.getPlanes()).thenReturn(planes); + + return mockImage; + } +} diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index ca9a4a47d32..e0254832899 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -4,9 +4,6 @@ package io.flutter.plugins.camera.media; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import android.graphics.ImageFormat; import android.media.Image; import java.nio.ByteBuffer; @@ -25,52 +22,10 @@ public void setUp() { this.imageStreamReaderUtils = new ImageStreamReaderUtils(); } - Image getImage(int imageWidth, int imageHeight, int padding) { - int rowStride = imageWidth + padding; - - int ySize = (rowStride * imageHeight) - padding; - int uSize = (ySize / 2) - (padding / 2); - int vSize = uSize; - - // Mock YUV image - Image mockImage = mock(Image.class); - when(mockImage.getWidth()).thenReturn(imageWidth); - when(mockImage.getHeight()).thenReturn(imageHeight); - when(mockImage.getFormat()).thenReturn(ImageFormat.YUV_420_888); - - // Mock planes. YUV images have 3 planes (Y, U, V). - Image.Plane planeY = mock(Image.Plane.class); - Image.Plane planeU = mock(Image.Plane.class); - Image.Plane planeV = mock(Image.Plane.class); - - // Y plane is width*height - // Row stride is generally == width but when there is padding it will - // be larger. - // Here we are adding 256 padding. - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(ySize)); - when(planeY.getRowStride()).thenReturn(rowStride); - when(planeY.getPixelStride()).thenReturn(1); - - // U and V planes are always the same sizes/values. - // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 - when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(uSize)); - when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(vSize)); - when(planeU.getRowStride()).thenReturn(rowStride); - when(planeV.getRowStride()).thenReturn(rowStride); - when(planeU.getPixelStride()).thenReturn(2); - when(planeV.getPixelStride()).thenReturn(2); - - // Add planes to image - Image.Plane[] planes = {planeY, planeU, planeV}; - when(mockImage.getPlanes()).thenReturn(planes); - - return mockImage; - } - /** Ensure that passing in an image with padding returns one without padding */ @Test public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { - Image mockImage = getImage(160, 120, 16); + Image mockImage = ImageStreamReaderTestUtils.getImage(160, 120, 16, ImageFormat.YUV_420_888); int imageWidth = mockImage.getWidth(); int imageHeight = mockImage.getHeight(); @@ -85,7 +40,7 @@ public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { /** Ensure that passing in an image without padding returns the same size */ @Test public void yuv420ThreePlanesToNV21_trimsPaddingWhenAbsent() { - Image mockImage = getImage(160, 120, 0); + Image mockImage = ImageStreamReaderTestUtils.getImage(160, 120, 0, ImageFormat.YUV_420_888); int imageWidth = mockImage.getWidth(); int imageHeight = mockImage.getHeight(); diff --git a/packages/camera/camera_android/pubspec.yaml b/packages/camera/camera_android/pubspec.yaml index 57d7f45f52d..09b19cb6a00 100644 --- a/packages/camera/camera_android/pubspec.yaml +++ b/packages/camera/camera_android/pubspec.yaml @@ -3,7 +3,7 @@ description: Android implementation of the camera plugin. repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_android issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.10.10+3 +version: 0.10.10+4 environment: sdk: ^3.6.0