From 4f8587f8e60b4698c3c207335f91a895b9b4a1f5 Mon Sep 17 00:00:00 2001
From: alanwutang11 <alanwumail11@gmail.com>
Date: Fri, 16 Dec 2022 13:23:44 -0800
Subject: [PATCH 1/2] use canvaskit toByteData for unsupported videoFrame
 formats

---
 .../lib/src/engine/canvaskit/image.dart       |  3 +-
 .../test/canvaskit/image_golden_test.dart     | 51 +++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart
index d9545a2e1bdeb..46d3e97d7f8ba 100644
--- a/lib/web_ui/lib/src/engine/canvaskit/image.dart
+++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart
@@ -376,7 +376,8 @@ class CkImage implements ui.Image, StackTraceDebugger {
     ui.ImageByteFormat format = ui.ImageByteFormat.rawRgba,
   }) {
     assert(_debugCheckIsNotDisposed());
-    if (videoFrame != null) {
+    // Web codecs do not support I420, I444, I422 videoFrame formats.
+    if (videoFrame != null && videoFrame!.format != 'I420' && videoFrame!.format != 'I444' && videoFrame!.format != 'I422') {
       return readPixelsFromVideoFrame(videoFrame!, format);
     } else {
       return _readPixelsFromSkImage(format);
diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart
index 08fbb721929ed..44ea1c74560b0 100644
--- a/lib/web_ui/test/canvaskit/image_golden_test.dart
+++ b/lib/web_ui/test/canvaskit/image_golden_test.dart
@@ -192,6 +192,57 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) {
       testCollector.collectNow();
     });
 
+    test('toByteData with decodeImageFromPixels on videoFrame formats', () async {
+      // This test ensures that toByteData() returns pixels that can be used by decodeImageFromPixels
+      // for the following videoFrame formats:
+      // [BGRX, I422, I420, I444, BGRA]
+      final DomResponse listingResponse = await httpFetch('/test_images/');
+      final List<String> testFiles = (await listingResponse.json() as List<dynamic>).cast<String>();
+
+      Future<ui.Image> testDecodeFromPixels(Uint8List pixels, int width, int height) async {
+        final Completer<ui.Image> completer = Completer<ui.Image>();
+        ui.decodeImageFromPixels(
+          pixels,
+          width,
+          height,
+          ui.PixelFormat.rgba8888,
+          (ui.Image image) {
+            completer.complete(image);
+          },
+        );
+        return completer.future;
+      }
+
+      // Sanity-check the test file list. If suddenly test files are moved or
+      // deleted, and the test server returns an empty list, or is missing some
+      // important test files, we want to know.
+      expect(testFiles, isNotEmpty);
+      expect(testFiles, contains(matches(RegExp(r'.*\.jpg'))));
+      expect(testFiles, contains(matches(RegExp(r'.*\.png'))));
+      expect(testFiles, contains(matches(RegExp(r'.*\.gif'))));
+      expect(testFiles, contains(matches(RegExp(r'.*\.webp'))));
+      expect(testFiles, contains(matches(RegExp(r'.*\.bmp'))));
+
+      for (final String testFile in testFiles) {
+        final DomResponse imageResponse = await httpFetch('/test_images/$testFile');
+        final Uint8List imageData = (await imageResponse.arrayBuffer() as ByteBuffer).asUint8List();
+        final ui.Codec codec = await skiaInstantiateImageCodec(imageData);
+        expect(codec.frameCount, greaterThan(0));
+        expect(codec.repetitionCount, isNotNull);
+
+        final ui.FrameInfo frame = await codec.getNextFrame();
+        final CkImage ckImage = frame.image as CkImage;
+        final ByteData imageBytes = await ckImage.toByteData();
+        expect(imageBytes.lengthInBytes, greaterThan(0));
+
+        final Uint8List pixels = imageBytes.buffer.asUint8List();
+        final ui.Image testImage = await testDecodeFromPixels(pixels, ckImage.width, ckImage.height);
+        expect(testImage, isNotNull);
+        codec.dispose();
+      }
+      // TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder.
+    }, skip: isFirefox || isSafari);
+
     test('CkImage.clone also clones the VideoFrame', () async {
       final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create(
         data: kAnimatedGif,

From baf863b35459d07ecd466bfe186daf30ba7e6161 Mon Sep 17 00:00:00 2001
From: alanwutang11 <alanwumail11@gmail.com>
Date: Sat, 17 Dec 2022 21:22:52 -0800
Subject: [PATCH 2/2] more descriptive comment

---
 lib/web_ui/lib/src/engine/canvaskit/image.dart | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart
index 46d3e97d7f8ba..0aec743b3ae86 100644
--- a/lib/web_ui/lib/src/engine/canvaskit/image.dart
+++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart
@@ -376,7 +376,8 @@ class CkImage implements ui.Image, StackTraceDebugger {
     ui.ImageByteFormat format = ui.ImageByteFormat.rawRgba,
   }) {
     assert(_debugCheckIsNotDisposed());
-    // Web codecs do not support I420, I444, I422 videoFrame formats.
+    // readPixelsFromVideoFrame currently does not convert I420, I444, I422
+    // videoFrame formats to RGBA
     if (videoFrame != null && videoFrame!.format != 'I420' && videoFrame!.format != 'I444' && videoFrame!.format != 'I422') {
       return readPixelsFromVideoFrame(videoFrame!, format);
     } else {