[Android] Cache GPU resources using HardwareBuffer's id as key

ci/licenses_golden/excluded_files

@@ -282,6 +282,7 @@
 ../../../flutter/shell/platform/android/external_view_embedder/surface_pool_unittests.cc
 ../../../flutter/shell/platform/android/flutter_shell_native_unittests.cc
 ../../../flutter/shell/platform/android/gradle.properties
+../../../flutter/shell/platform/android/image_lru_unittests.cc
 ../../../flutter/shell/platform/android/jni/jni_mock_unittest.cc
 ../../../flutter/shell/platform/android/platform_view_android_delegate/platform_view_android_delegate_unittests.cc
 ../../../flutter/shell/platform/android/platform_view_android_unittests.cc

ci/licenses_golden/licenses_flutter

@@ -6347,6 +6347,8 @@ ORIGIN: ../../../flutter/shell/platform/android/image_external_texture.h + ../..
 ORIGIN: ../../../flutter/shell/platform/android/image_external_texture_gl.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/shell/platform/android/image_external_texture_gl.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/shell/platform/android/image_external_texture_vk.h + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/shell/platform/android/image_lru.cc + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/shell/platform/android/image_lru.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/shell/platform/android/io/flutter/BuildConfig.java + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/shell/platform/android/io/flutter/Log.java + ../../../flutter/LICENSE
@@ -9203,6 +9205,8 @@ FILE: ../../../flutter/shell/platform/android/image_external_texture_gl.cc
 FILE: ../../../flutter/shell/platform/android/image_external_texture_gl.h
 FILE: ../../../flutter/shell/platform/android/image_external_texture_vk.cc
 FILE: ../../../flutter/shell/platform/android/image_external_texture_vk.h
+FILE: ../../../flutter/shell/platform/android/image_lru.cc
+FILE: ../../../flutter/shell/platform/android/image_lru.h
 FILE: ../../../flutter/shell/platform/android/io/flutter/BuildConfig.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/FlutterInjector.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/Log.java

shell/platform/android/BUILD.gn

@@ -47,6 +47,7 @@ executable("flutter_shell_native_unittests") {
     "android_shell_holder_unittests.cc",
     "apk_asset_provider_unittests.cc",
     "flutter_shell_native_unittests.cc",
+    "image_lru_unittests.cc",
     "platform_view_android_unittests.cc",
   ]
   public_configs = [ "//flutter:config" ]
@@ -111,6 +112,8 @@ source_set("flutter_shell_native_src") {
     "image_external_texture_gl.h",
     "image_external_texture_vk.cc",
     "image_external_texture_vk.h",
+    "image_lru.cc",
+    "image_lru.h",
     "library_loader.cc",
     "ndk_helpers.cc",
     "ndk_helpers.h",

shell/platform/android/image_external_texture.cc

@@ -25,6 +25,7 @@ void ImageExternalTexture::Paint(PaintContext& context,
   if (state_ == AttachmentState::kDetached) {
     return;
   }
+  latest_bounds_ = bounds;
   Attach(context);
   const bool should_process_frame = !freeze;
   if (should_process_frame) {
@@ -61,6 +62,7 @@ void ImageExternalTexture::OnGrContextCreated() {
 void ImageExternalTexture::OnGrContextDestroyed() {
   if (state_ == AttachmentState::kAttached) {
     dl_image_.reset();
+    image_lru_.Clear();
     Detach();
   }
   state_ = AttachmentState::kDetached;

shell/platform/android/image_external_texture.h

@@ -7,6 +7,7 @@
 
 #include "flutter/common/graphics/texture.h"
 #include "flutter/fml/logging.h"
+#include "flutter/shell/platform/android/image_lru.h"
 #include "flutter/shell/platform/android/jni/platform_view_android_jni.h"
 #include "flutter/shell/platform/android/platform_view_android_jni_impl.h"
 
@@ -58,11 +59,13 @@ class ImageExternalTexture : public flutter::Texture {
 
   fml::jni::ScopedJavaGlobalRef<jobject> image_texture_entry_;
   std::shared_ptr<PlatformViewAndroidJNI> jni_facade_;
+  SkRect latest_bounds_ = SkRect::MakeEmpty();
+  fml::jni::ScopedJavaGlobalRef<jobject> latest_android_image_;
 
   enum class AttachmentState { kUninitialized, kAttached, kDetached };
   AttachmentState state_ = AttachmentState::kUninitialized;
-
   sk_sp<flutter::DlImage> dl_image_;
+  ImageLRU image_lru_ = ImageLRU();
 
   FML_DISALLOW_COPY_AND_ASSIGN(ImageExternalTexture);
 };

shell/platform/android/image_external_texture_gl.cc

@@ -8,20 +8,13 @@
 #include <android/sensor.h>
 
 #include "flutter/common/graphics/texture.h"
-#include "flutter/display_list/effects/dl_color_source.h"
-#include "flutter/flow/layers/layer.h"
 #include "flutter/impeller/core/formats.h"
 #include "flutter/impeller/display_list/dl_image_impeller.h"
-#include "flutter/impeller/renderer/backend/gles/texture_gles.h"
 #include "flutter/impeller/toolkit/egl/image.h"
 #include "flutter/impeller/toolkit/gles/texture.h"
 #include "flutter/shell/platform/android/ndk_helpers.h"
 #include "third_party/skia/include/core/SkAlphaType.h"
-#include "third_party/skia/include/core/SkColorSpace.h"
 #include "third_party/skia/include/core/SkColorType.h"
-#include "third_party/skia/include/core/SkImage.h"
-#include "third_party/skia/include/gpu/GrBackendSurface.h"
-#include "third_party/skia/include/gpu/GrDirectContext.h"
 #include "third_party/skia/include/gpu/ganesh/SkImageGanesh.h"
 #include "third_party/skia/include/gpu/ganesh/gl/GrGLBackendSurface.h"
 #include "third_party/skia/include/gpu/gl/GrGLTypes.h"
@@ -36,40 +29,61 @@ ImageExternalTextureGL::ImageExternalTextureGL(
 
 void ImageExternalTextureGL::Attach(PaintContext& context) {
   if (state_ == AttachmentState::kUninitialized) {
-    if (!android_image_.is_null()) {
-      JavaLocalRef hardware_buffer = HardwareBufferFor(android_image_);
-      AHardwareBuffer* hardware_buffer_ahw =
-          AHardwareBufferFor(hardware_buffer);
-      egl_image_ = CreateEGLImage(hardware_buffer_ahw);
+    if (!latest_android_image_.is_null() && !latest_bounds_.isEmpty()) {
+      // After detach the cache of textures will have been cleared. If
+      // there is an android image we must populate it now so that the
+      // first frame isn't blank.
+      JavaLocalRef hardware_buffer = HardwareBufferFor(latest_android_image_);
+      UpdateImage(hardware_buffer, context);
       CloseHardwareBuffer(hardware_buffer);
     }
     state_ = AttachmentState::kAttached;
   }
 }
 
-void ImageExternalTextureGL::Detach() {
-  egl_image_.reset();
+void ImageExternalTextureGL::UpdateImage(JavaLocalRef& hardware_buffer,
+                                         PaintContext& context) {
+  AHardwareBuffer* latest_hardware_buffer = AHardwareBufferFor(hardware_buffer);
+  HardwareBufferKey key =
+      flutter::NDKHelpers::AHardwareBuffer_getId(latest_hardware_buffer);
+  auto existing_image = image_lru_.FindImage(key);
+  if (existing_image != nullptr) {
+    dl_image_ = existing_image;
+    return;
+  }
+
+  auto egl_image = CreateEGLImage(latest_hardware_buffer);
+  if (!egl_image.is_valid()) {
+    return;
+  }
+
+  dl_image_ = CreateDlImage(context, latest_bounds_, key, std::move(egl_image));
+  gl_entries_.erase(image_lru_.AddImage(dl_image_, key));
 }
 
-bool ImageExternalTextureGL::MaybeSwapImages() {
+void ImageExternalTextureGL::ProcessFrame(PaintContext& context,
+                                          const SkRect& bounds) {
   JavaLocalRef image = AcquireLatestImage();
   if (image.is_null()) {
-    return false;
+    return;
   }
+  JavaLocalRef hardware_buffer = HardwareBufferFor(image);
+  UpdateImage(hardware_buffer, context);
+  CloseHardwareBuffer(hardware_buffer);
+
   // NOTE: In the following code it is important that old_android_image is
   // not closed until after the update of egl_image_ otherwise the image might
   // be closed before the old EGLImage referencing it has been deleted. After
   // an image is closed the underlying HardwareBuffer may be recycled and used
   // for a future frame.
-  JavaLocalRef old_android_image(android_image_);
-  android_image_.Reset(image);
-  JavaLocalRef hardware_buffer = HardwareBufferFor(image);
-  egl_image_ = CreateEGLImage(AHardwareBufferFor(hardware_buffer));
-  CloseHardwareBuffer(hardware_buffer);
-  // IMPORTANT: We only close the old image after egl_image_ stops referencing
-  // it.
+  JavaLocalRef old_android_image(latest_android_image_);
+  latest_android_image_.Reset(image);
   CloseImage(old_android_image);
-  return true;
+}
+
+void ImageExternalTextureGL::Detach() {
+  image_lru_.Clear();
+  gl_entries_.clear();
 }
 
 impeller::UniqueEGLImageKHR ImageExternalTextureGL::CreateEGLImage(
@@ -110,26 +124,11 @@ void ImageExternalTextureGLSkia::Attach(PaintContext& context) {
     // After this call state_ will be AttachmentState::kAttached and egl_image_
     // will have been created if we still have an Image associated with us.
     ImageExternalTextureGL::Attach(context);
-    GLuint texture_name;
-    glGenTextures(1, &texture_name);
-    texture_.reset(impeller::GLTexture{texture_name});
   }
 }
 
 void ImageExternalTextureGLSkia::Detach() {
   ImageExternalTextureGL::Detach();
-  texture_.reset();
-}
-
-void ImageExternalTextureGLSkia::ProcessFrame(PaintContext& context,
-                                              const SkRect& bounds) {
-  const bool swapped = MaybeSwapImages();
-  if (!swapped && !egl_image_.is_valid()) {
-    // Nothing to do.
-    return;
-  }
-  BindImageToTexture(egl_image_, texture_.get().texture_name);
-  dl_image_ = CreateDlImage(context, bounds);
 }
 
 void ImageExternalTextureGLSkia::BindImageToTexture(
@@ -145,11 +144,22 @@ void ImageExternalTextureGLSkia::BindImageToTexture(
 
 sk_sp<flutter::DlImage> ImageExternalTextureGLSkia::CreateDlImage(
     PaintContext& context,
-    const SkRect& bounds) {
-  GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES,
-                                 texture_.get().texture_name, GL_RGBA8_OES};
+    const SkRect& bounds,
+    HardwareBufferKey id,
+    impeller::UniqueEGLImageKHR&& egl_image) {
+  GLuint texture_name;
+  glGenTextures(1, &texture_name);
+  auto gl_texture = impeller::GLTexture{texture_name};
+  impeller::UniqueGLTexture unique_texture;
+  unique_texture.reset(gl_texture);
+
+  BindImageToTexture(egl_image, unique_texture.get().texture_name);
+  GrGLTextureInfo textureInfo = {
+      GL_TEXTURE_EXTERNAL_OES, unique_texture.get().texture_name, GL_RGBA8_OES};
   auto backendTexture =
       GrBackendTextures::MakeGL(1, 1, skgpu::Mipmapped::kNo, textureInfo);
+  gl_entries_[id] = GlEntry{.egl_image = std::move(egl_image),
+                            .texture = std::move(unique_texture)};
   return DlImage::Make(SkImages::BorrowTextureFrom(
       context.gr_context, backendTexture, kTopLeft_GrSurfaceOrigin,
       kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr));
@@ -171,19 +181,11 @@ void ImageExternalTextureGLImpeller::Attach(PaintContext& context) {
   }
 }
 
-void ImageExternalTextureGLImpeller::ProcessFrame(PaintContext& context,
-                                                  const SkRect& bounds) {
-  const bool swapped = MaybeSwapImages();
-  if (!swapped && !egl_image_.is_valid()) {
-    // Nothing to do.
-    return;
-  }
-  dl_image_ = CreateDlImage(context, bounds);
-}
-
 sk_sp<flutter::DlImage> ImageExternalTextureGLImpeller::CreateDlImage(
     PaintContext& context,
-    const SkRect& bounds) {
+    const SkRect& bounds,
+    HardwareBufferKey id,
+    impeller::UniqueEGLImageKHR&& egl_image) {
   impeller::TextureDescriptor desc;
   desc.type = impeller::TextureType::kTextureExternalOES;
   desc.storage_mode = impeller::StorageMode::kDevicePrivate;
@@ -201,7 +203,10 @@ sk_sp<flutter::DlImage> ImageExternalTextureGLImpeller::CreateDlImage(
   }
   // Associate the hardware buffer image with the texture.
   glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES,
-                               (GLeglImageOES)egl_image_.get().image);
+                               (GLeglImageOES)egl_image.get().image);
+  gl_entries_[id] = GlEntry{
+      .egl_image = std::move(egl_image),
+  };
   return impeller::DlImageImpeller::Make(texture);
 }
 

shell/platform/android/image_external_texture_gl.h

@@ -5,6 +5,8 @@
 #ifndef FLUTTER_SHELL_PLATFORM_ANDROID_IMAGE_EXTERNAL_TEXTURE_GL_H_
 #define FLUTTER_SHELL_PLATFORM_ANDROID_IMAGE_EXTERNAL_TEXTURE_GL_H_
 
+#include <unordered_map>
+
 #include "flutter/fml/platform/android/scoped_java_ref.h"
 #include "flutter/shell/platform/android/image_external_texture.h"
 
@@ -30,14 +32,26 @@ class ImageExternalTextureGL : public ImageExternalTexture {
  protected:
   void Attach(PaintContext& context) override;
   void Detach() override;
+  void ProcessFrame(PaintContext& context, const SkRect& bounds) override;
+  void UpdateImage(JavaLocalRef& hardware_buffer, PaintContext& context);
+
+  virtual sk_sp<flutter::DlImage> CreateDlImage(
+      PaintContext& context,
+      const SkRect& bounds,
+      HardwareBufferKey id,
+      impeller::UniqueEGLImageKHR&& egl_image) = 0;
 
-  // Returns true if a new image was acquired and android_image_ and egl_image_
-  // were updated.
-  bool MaybeSwapImages();
   impeller::UniqueEGLImageKHR CreateEGLImage(AHardwareBuffer* buffer);
 
-  fml::jni::ScopedJavaGlobalRef<jobject> android_image_;
-  impeller::UniqueEGLImageKHR egl_image_;
+  struct GlEntry {
+    impeller::UniqueEGLImageKHR egl_image;
+    impeller::UniqueGLTexture texture;
+  };
+
+  // Each GL entry is keyed off of the currently active
+  // hardware buffers and evicted when the hardware buffer
+  // is removed from the LRU cache.
+  std::unordered_map<HardwareBufferKey, GlEntry> gl_entries_;
 
   FML_DISALLOW_COPY_AND_ASSIGN(ImageExternalTextureGL);
 };
@@ -53,13 +67,14 @@ class ImageExternalTextureGLSkia : public ImageExternalTextureGL {
  private:
   void Attach(PaintContext& context) override;
   void Detach() override;
-  void ProcessFrame(PaintContext& context, const SkRect& bounds) override;
 
   void BindImageToTexture(const impeller::UniqueEGLImageKHR& image, GLuint tex);
-  sk_sp<flutter::DlImage> CreateDlImage(PaintContext& context,
-                                        const SkRect& bounds);
 
-  impeller::UniqueGLTexture texture_;
+  sk_sp<flutter::DlImage> CreateDlImage(
+      PaintContext& context,
+      const SkRect& bounds,
+      HardwareBufferKey id,
+      impeller::UniqueEGLImageKHR&& egl_image) override;
 
   FML_DISALLOW_COPY_AND_ASSIGN(ImageExternalTextureGLSkia);
 };
@@ -75,11 +90,13 @@ class ImageExternalTextureGLImpeller : public ImageExternalTextureGL {
 
  private:
   void Attach(PaintContext& context) override;
-  void ProcessFrame(PaintContext& context, const SkRect& bounds) override;
   void Detach() override;
 
-  sk_sp<flutter::DlImage> CreateDlImage(PaintContext& context,
-                                        const SkRect& bounds);
+  sk_sp<flutter::DlImage> CreateDlImage(
+      PaintContext& context,
+      const SkRect& bounds,
+      HardwareBufferKey id,
+      impeller::UniqueEGLImageKHR&& egl_image) override;
 
   const std::shared_ptr<impeller::ContextGLES> impeller_context_;
 

shell/platform/android/image_external_texture_vk.cc

@@ -1,5 +1,6 @@
 
 #include "flutter/shell/platform/android/image_external_texture_vk.h"
+#include <cstdint>
 
 #include "flutter/impeller/core/formats.h"
 #include "flutter/impeller/core/texture_descriptor.h"
@@ -37,14 +38,26 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context,
   if (image.is_null()) {
     return;
   }
-  JavaLocalRef old_android_image(android_image_);
-  android_image_.Reset(image);
-  JavaLocalRef hardware_buffer = HardwareBufferFor(android_image_);
+  JavaLocalRef old_android_image(latest_android_image_);
+  latest_android_image_.Reset(image);
+  JavaLocalRef hardware_buffer = HardwareBufferFor(latest_android_image_);
   AHardwareBuffer* latest_hardware_buffer = AHardwareBufferFor(hardware_buffer);
 
   AHardwareBuffer_Desc hb_desc = {};
   flutter::NDKHelpers::AHardwareBuffer_describe(latest_hardware_buffer,
                                                 &hb_desc);
+  HardwareBufferKey key =
+      flutter::NDKHelpers::AHardwareBuffer_getId(latest_hardware_buffer);
+  auto existing_image = image_lru_.FindImage(key);
+  if (existing_image != nullptr) {
+    dl_image_ = existing_image;
+
+    CloseHardwareBuffer(hardware_buffer);
+    // IMPORTANT: We have just received a new frame to display so close the
+    // previous Java Image so that it is recycled and used for a future frame.
+    CloseImage(old_android_image);
+    return;
+  }
 
   impeller::TextureDescriptor desc;
   desc.storage_mode = impeller::StorageMode::kDevicePrivate;
@@ -88,9 +101,10 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context,
   }
 
   dl_image_ = impeller::DlImageImpeller::Make(texture);
+  image_lru_.AddImage(dl_image_, key);
   CloseHardwareBuffer(hardware_buffer);
-  // IMPORTANT: We only close the old image after texture stops referencing
-  // it.
+  // IMPORTANT: We have just received a new frame to display so close the
+  // previous Java Image so that it is recycled and used for a future frame.
   CloseImage(old_android_image);
 }