Reland "[canvaskit] Further improve overlay optimization by splitting pictures"

ci/licenses_golden/licenses_flutter

@@ -43589,6 +43589,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.da
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_scene_builder.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_tree.dart + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_visitor.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/mask_filter.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/n_way_canvas.dart + ../../../flutter/LICENSE
@@ -46469,6 +46470,7 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_scene_builder.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_tree.dart
+FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/layer_visitor.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/mask_filter.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart
 FILE: ../../../flutter/lib/web_ui/lib/src/engine/canvaskit/n_way_canvas.dart

lib/web_ui/lib/src/engine.dart

@@ -33,6 +33,7 @@ export 'engine/canvaskit/image_web_codecs.dart';
 export 'engine/canvaskit/layer.dart';
 export 'engine/canvaskit/layer_scene_builder.dart';
 export 'engine/canvaskit/layer_tree.dart';
+export 'engine/canvaskit/layer_visitor.dart';
 export 'engine/canvaskit/mask_filter.dart';
 export 'engine/canvaskit/multi_surface_rasterizer.dart';
 export 'engine/canvaskit/n_way_canvas.dart';

lib/web_ui/lib/src/engine/canvaskit/embedded_views.dart

@@ -14,6 +14,7 @@ import '../svg.dart';
 import '../util.dart';
 import '../vector_math.dart';
 import 'canvas.dart';
+import 'layer.dart';
 import 'overlay_scene_optimizer.dart';
 import 'painting.dart';
 import 'path.dart';
@@ -66,6 +67,9 @@ class HtmlViewEmbedder {
   /// Returns the most recent rendering. Only used in tests.
   Rendering get debugActiveRendering => _activeRendering;
 
+  /// If [debugOverlayOptimizationBounds] is true, this canvas will draw
+  /// semitransparent rectangles showing the computed bounds of the platform
+  /// views and pictures in the scene.
   DisplayCanvas? debugBoundsCanvas;
 
   /// The size of the frame, in physical pixels.
@@ -75,27 +79,23 @@ class HtmlViewEmbedder {
     _frameSize = size;
   }
 
-  /// Returns a list of canvases which will be overlaid on top of the "base"
-  /// canvas after a platform view is composited into the scene.
-  ///
-  /// The engine asks for the overlay canvases immediately before the paint
-  /// phase, after the preroll phase. In the preroll phase we must be
-  /// conservative and assume that every platform view which is prerolled is
-  /// also composited, and therefore requires an overlay canvas. However, not
-  /// every platform view which is prerolled ends up being composited (it may be
-  /// clipped out and not actually drawn). This means that we may end up
-  /// overallocating canvases. This isn't a problem in practice, however, as
-  /// unused recording canvases are simply deleted at the end of the frame.
-  Iterable<CkCanvas> getOverlayCanvases() {
-    return _context.pictureRecordersCreatedDuringPreroll
+  /// Returns a list of recording canvases which the pictures in the upcoming
+  /// paint step will be drawn into. These recording canvases are combined into
+  /// an N-way canvas for the rasterizer to record clip and transform operations
+  /// during the measure step.
+  Iterable<CkCanvas> getPictureCanvases() {
+    return _context.measuringPictureRecorders.values
         .map((CkPictureRecorder r) => r.recordingCanvas!);
   }
 
-  void prerollCompositeEmbeddedView(int viewId, EmbeddedViewParams params) {
-    final CkPictureRecorder pictureRecorder = CkPictureRecorder();
-    pictureRecorder.beginRecording(ui.Offset.zero & _frameSize.toSize());
-    _context.pictureRecordersCreatedDuringPreroll.add(pictureRecorder);
+  /// Returns a list of canvases for the optimized rendering. These are used in
+  /// the paint step.
+  Iterable<CkCanvas> getOptimizedCanvases() {
+    return _context.optimizedCanvasRecorders!
+        .map((CkPictureRecorder r) => r.recordingCanvas!);
+  }
 
+  void prerollCompositeEmbeddedView(int viewId, EmbeddedViewParams params) {
     // Do nothing if the params didn't change.
     if (_currentCompositionParams[viewId] == params) {
       // If the view was prerolled but not composited, then it needs to be
@@ -109,30 +109,38 @@ class HtmlViewEmbedder {
     _viewsToRecomposite.add(viewId);
   }
 
+  /// Record that a picture recorder is needed for [picture] to be measured.
+  void prerollPicture(PictureLayer picture) {
+    final CkPictureRecorder pictureRecorder = CkPictureRecorder();
+    pictureRecorder.beginRecording(ui.Offset.zero & _frameSize.toSize());
+    _context.measuringPictureRecorders[picture] = pictureRecorder;
+  }
+
+  /// Returns the canvas that was created to measure [picture].
+  CkCanvas getMeasuringCanvasFor(PictureLayer picture) {
+    return _context.measuringPictureRecorders[picture]!.recordingCanvas!;
+  }
+
+  /// Adds the picture recorder associated with [picture] to the unoptimized
+  /// scene.
+  void addPictureToUnoptimizedScene(PictureLayer picture) {
+    final CkPictureRecorder recorder =
+        _context.measuringPictureRecorders[picture]!;
+    _context.sceneElements.add(PictureSceneElement(picture, recorder));
+  }
+
   /// Prepares to composite [viewId].
-  ///
-  /// If this returns a [CkCanvas], then that canvas should be the new leaf
-  /// node. Otherwise, keep the same leaf node.
-  CkCanvas? compositeEmbeddedView(int viewId) {
+  void compositeEmbeddedView(int viewId) {
     // Ensure platform view with `viewId` is injected into the `rasterizer.view`.
     rasterizer.view.dom.injectPlatformView(viewId);
 
-    final int overlayIndex = _context.viewCount;
     _compositionOrder.add(viewId);
-    _context.viewCount++;
-
-    CkPictureRecorder? recorderToUseForRendering;
-    if (overlayIndex < _context.pictureRecordersCreatedDuringPreroll.length) {
-      recorderToUseForRendering =
-          _context.pictureRecordersCreatedDuringPreroll[overlayIndex];
-      _context.pictureRecorders.add(recorderToUseForRendering);
-    }
+    _context.sceneElements.add(PlatformViewSceneElement(viewId));
 
     if (_viewsToRecomposite.contains(viewId)) {
       _compositeWithParams(viewId, _currentCompositionParams[viewId]!);
       _viewsToRecomposite.remove(viewId);
     }
-    return recorderToUseForRendering?.recordingCanvas;
   }
 
   void _compositeWithParams(int platformViewId, EmbeddedViewParams params) {
@@ -355,14 +363,57 @@ class HtmlViewEmbedder {
     sceneHost.append(_svgPathDefs!);
   }
 
-  Future<void> submitFrame(CkPicture basePicture) async {
-    final List<CkPicture> pictures = <CkPicture>[basePicture];
-    for (final CkPictureRecorder recorder in _context.pictureRecorders) {
-      pictures.add(recorder.endRecording());
-    }
+  /// Optimizes the scene to use the fewest possible canvases. This sets up
+  /// the final paint pass to paint the pictures into the optimized canvases.
+  void optimizeRendering() {
+    final Map<CkPicture, PictureLayer> scenePictureToRawPicture =
+        <CkPicture, PictureLayer>{};
+    final Iterable<SceneElement> unoptimizedRendering =
+        _context.sceneElements.map<SceneElement>((SceneElement element) {
+      if (element is PictureSceneElement) {
+        final CkPicture scenePicture = element.pictureRecorder.endRecording();
+        if (scenePicture.cullRect.isEmpty) {
+          element.picture.isCulled = true;
+        }
+        element.scenePicture = scenePicture;
+        scenePictureToRawPicture[scenePicture] = element.picture;
+        return element;
+      } else {
+        return element;
+      }
+    });
     Rendering rendering = createOptimizedRendering(
-        pictures, _compositionOrder, _currentCompositionParams);
+        unoptimizedRendering, _currentCompositionParams);
     rendering = _modifyRenderingForMaxCanvases(rendering);
+    _context.optimizedRendering = rendering;
+    // Create new picture recorders for the optimized render canvases and record
+    // which pictures go in which canvas.
+    final List<CkPictureRecorder> optimizedCanvasRecorders =
+        <CkPictureRecorder>[];
+    final Map<PictureLayer, CkPictureRecorder> pictureToOptimizedCanvasMap =
+        <PictureLayer, CkPictureRecorder>{};
+    for (final RenderingRenderCanvas renderCanvas in rendering.canvases) {
+      final CkPictureRecorder pictureRecorder = CkPictureRecorder();
+      pictureRecorder.beginRecording(ui.Offset.zero & _frameSize.toSize());
+      optimizedCanvasRecorders.add(pictureRecorder);
+      for (final CkPicture picture in renderCanvas.pictures) {
+        pictureToOptimizedCanvasMap[scenePictureToRawPicture[picture]!] =
+            pictureRecorder;
+      }
+    }
+    _context.optimizedCanvasRecorders = optimizedCanvasRecorders;
+    _context.pictureToOptimizedCanvasMap = pictureToOptimizedCanvasMap;
+  }
+
+  /// Returns the canvas that this picture layer should draw into in the
+  /// optimized scene.
+  CkCanvas getOptimizedCanvasFor(PictureLayer picture) {
+    assert(_context.optimizedRendering != null);
+    return _context.pictureToOptimizedCanvasMap![picture]!.recordingCanvas!;
+  }
+
+  Future<void> submitFrame() async {
+    final Rendering rendering = _context.optimizedRendering!;
     _updateDomForNewRendering(rendering);
     if (rendering.equalsForRendering(_activeRendering)) {
       // Copy the display canvases to the new rendering.
@@ -375,13 +426,17 @@ class HtmlViewEmbedder {
     _activeRendering = rendering;
 
     final List<RenderingRenderCanvas> renderCanvases = rendering.canvases;
+    int renderCanvasIndex = 0;
     for (final RenderingRenderCanvas renderCanvas in renderCanvases) {
+      final CkPicture renderPicture = _context
+          .optimizedCanvasRecorders![renderCanvasIndex++]
+          .endRecording();
       await rasterizer.rasterizeToCanvas(
-          renderCanvas.displayCanvas!, renderCanvas.pictures);
+          renderCanvas.displayCanvas!, <CkPicture>[renderPicture]);
     }
 
     for (final CkPictureRecorder recorder
-        in _context.pictureRecordersCreatedDuringPreroll) {
+        in _context.measuringPictureRecorders.values) {
       if (recorder.isRecording) {
         recorder.endRecording();
       }
@@ -393,11 +448,11 @@ class HtmlViewEmbedder {
       debugBoundsCanvas ??= rasterizer.displayFactory.getCanvas();
       final CkPictureRecorder boundsRecorder = CkPictureRecorder();
       final CkCanvas boundsCanvas = boundsRecorder.beginRecording(
-          ui.Rect.fromLTWH(
-        0,
-        0,
-        _frameSize.width.toDouble(),
-        _frameSize.height.toDouble(),
+        ui.Rect.fromLTWH(
+          0,
+          0,
+          _frameSize.width.toDouble(),
+          _frameSize.height.toDouble(),
         ),
       );
       final CkPaint platformViewBoundsPaint = CkPaint()
@@ -903,20 +958,45 @@ class MutatorsStack extends Iterable<Mutator> {
   Iterable<Mutator> get reversed => _mutators;
 }
 
-/// The state for the current frame.
-class EmbedderFrameContext {
-  /// Picture recorders which were created during the preroll phase.
-  ///
-  /// These picture recorders will be "claimed" in the paint phase by platform
-  /// views being composited into the scene.
-  final List<CkPictureRecorder> pictureRecordersCreatedDuringPreroll =
-      <CkPictureRecorder>[];
+sealed class SceneElement {}
 
-  /// Picture recorders which were actually used in the paint phase.
-  ///
-  /// This is a subset of [_pictureRecordersCreatedDuringPreroll].
-  final List<CkPictureRecorder> pictureRecorders = <CkPictureRecorder>[];
+class PictureSceneElement extends SceneElement {
+  PictureSceneElement(this.picture, this.pictureRecorder);
+
+  final PictureLayer picture;
+  final CkPictureRecorder pictureRecorder;
+
+  /// The picture as it would be painted in the final scene, with clips and
+  /// transforms applied. This is set by [optimizeRendering].
+  CkPicture? scenePicture;
+}
 
-  /// The number of platform views in this frame.
-  int viewCount = 0;
+class PlatformViewSceneElement extends SceneElement {
+  PlatformViewSceneElement(this.viewId);
+
+  final int viewId;
+}
+
+/// The state for the current frame.
+class EmbedderFrameContext {
+  /// Picture recorders which were created d the final bounds of the picture in the scene.
+  final Map<PictureLayer, CkPictureRecorder> measuringPictureRecorders =
+      <PictureLayer, CkPictureRecorder>{};
+
+  /// List of picture recorders and platform view ids in the order they were
+  /// painted.
+  final List<SceneElement> sceneElements = <SceneElement>[];
+
+  /// The optimized rendering for this frame. This is set by calling
+  /// [optimizeRendering].
+  Rendering? optimizedRendering;
+
+  /// The picture recorders for the optimized rendering. This is set by calling
+  /// [optimizeRendering].
+  List<CkPictureRecorder>? optimizedCanvasRecorders;
+
+  /// A map from the original PictureLayer to the picture recorder it should go
+  /// into in the optimized rendering. This is set by calling
+  /// [optimizedRendering].
+  Map<PictureLayer, CkPictureRecorder>? pictureToOptimizedCanvasMap;
 }