[Android] Cache GPU resources using HardwareBuffer's id as key

shell/platform/android/image_external_texture.cc

@@ -59,10 +59,61 @@ void ImageExternalTexture::OnGrContextCreated() {
   state_ = AttachmentState::kUninitialized;
 }
 
+sk_sp<flutter::DlImage> ImageExternalTexture::FindImage(uint64_t key) {
+  for (size_t i = 0u; i < kImageReaderSwapchainSize; i++) {
+    if (images_[i].key == key) {
+      auto result = images_[i].image;
+      UpdateKey(result, key);
+      return result;
+    }
+  }
+  return nullptr;
+}
+
+void ImageExternalTexture::UpdateKey(const sk_sp<flutter::DlImage>& image,
+                                     uint64_t key) {
+  if (images_[0].key == key) {
+    return;
+  }
+  size_t i = 1u;
+  for (; i < kImageReaderSwapchainSize; i++) {
+    if (images_[i].key == key) {
+      break;
+    }
+  }
+  for (auto j = i; j > 0; j--) {
+    images_[j] = images_[j - 1];
+  }
+}
+
+void ImageExternalTexture::AddImage(const sk_sp<flutter::DlImage>& image,
+                                    uint64_t key) {
+  uint64_t lru_key = images_[kImageReaderSwapchainSize - 1].key;
+  bool updated_image = false;
+  for (size_t i = 0u; i < kImageReaderSwapchainSize; i++) {
+    if (images_[i].key == lru_key) {
+      updated_image = true;
+      images_[i] = LRUImage{.key = key, .image = image};
+      break;
+    }
+  }
+  if (!updated_image) {
+    images_[0] = LRUImage{.key = key, .image = image};
+  }
+  UpdateKey(image, key);
+}
+
+void ImageExternalTexture::ResetCache() {
+  for (size_t i = 0u; i < kImageReaderSwapchainSize; i++) {
+    images_[i] = {.key = 0u, .image = nullptr};
+  }
+}
+
 // Implementing flutter::ContextListener.
 void ImageExternalTexture::OnGrContextDestroyed() {
   if (state_ == AttachmentState::kAttached) {
     dl_image_.reset();
+    ResetCache();
     Detach();
   }
   state_ = AttachmentState::kDetached;

shell/platform/android/image_external_texture.h

@@ -15,6 +15,13 @@
 
 namespace flutter {
 
+// This value needs to be larger than the number of swapchain images
+// that a typical image reader will produce to ensure that we effectively
+// cache. If the value is too small, we will unnecessarily churn through
+// images, while if it is too large we may retain images longer than
+// necessary.
+static constexpr size_t kImageReaderSwapchainSize = 6u;
+
 // External texture peered to a sequence of android.hardware.HardwareBuffers.
 //
 class ImageExternalTexture : public flutter::Texture {
@@ -49,6 +56,16 @@ class ImageExternalTexture : public flutter::Texture {
   virtual void Detach() = 0;
   virtual void ProcessFrame(PaintContext& context, const SkRect& bounds) = 0;
 
+  sk_sp<flutter::DlImage> FindImage(uint64_t key);
+  void UpdateKey(const sk_sp<flutter::DlImage>& image, uint64_t key);
+  void AddImage(const sk_sp<flutter::DlImage>& image, uint64_t key);
+  void ResetCache();
+
+  struct LRUImage {
+    uint64_t key;
+    sk_sp<flutter::DlImage> image;
+  };
+
   JavaLocalRef AcquireLatestImage();
   void CloseImage(const fml::jni::JavaRef<jobject>& image);
   JavaLocalRef HardwareBufferFor(const fml::jni::JavaRef<jobject>& image);
@@ -62,6 +79,11 @@ class ImageExternalTexture : public flutter::Texture {
   enum class AttachmentState { kUninitialized, kAttached, kDetached };
   AttachmentState state_ = AttachmentState::kUninitialized;
   bool new_frame_ready_ = false;
+  std::array<LRUImage, kImageReaderSwapchainSize> images_ = {
+      LRUImage{.key = 0, .image = nullptr},
+      LRUImage{.key = 0, .image = nullptr},
+      LRUImage{.key = 0, .image = nullptr},
+  };
 
   sk_sp<flutter::DlImage> dl_image_;
 

shell/platform/android/image_external_texture_gl.cc

@@ -47,15 +47,13 @@ void ImageExternalTextureGL::Attach(PaintContext& context) {
   }
 }
 
-void ImageExternalTextureGL::Detach() {
-  egl_image_.reset();
-}
-
-bool ImageExternalTextureGL::MaybeSwapImages() {
+void ImageExternalTextureGL::ProcessFrame(PaintContext& context,
+                                          const SkRect& bounds) {
   JavaLocalRef image = AcquireLatestImage();
   if (image.is_null()) {
-    return false;
+    return;
   }
+
   // NOTE: In the following code it is important that old_android_image is
   // not closed until after the update of egl_image_ otherwise the image might
   // be closed before the old EGLImage referencing it has been deleted. After
@@ -64,12 +62,35 @@ bool ImageExternalTextureGL::MaybeSwapImages() {
   JavaLocalRef old_android_image(android_image_);
   android_image_.Reset(image);
   JavaLocalRef hardware_buffer = HardwareBufferFor(image);
-  egl_image_ = CreateEGLImage(AHardwareBufferFor(hardware_buffer));
+  AHardwareBuffer* latest_hardware_buffer = AHardwareBufferFor(hardware_buffer);
+  auto key = flutter::NDKHelpers::AHardwareBuffer_getId(latest_hardware_buffer);
+  auto existing_image = FindImage(key);
+  if (existing_image != nullptr) {
+    dl_image_ = existing_image;
+
+    CloseHardwareBuffer(hardware_buffer);
+    // IMPORTANT: We have just received a new frame to display so close the
+    // previous Java Image so that it is recycled and used for a future frame.
+    CloseImage(old_android_image);
+    return;
+  }
+
+  egl_image_ = CreateEGLImage(latest_hardware_buffer);
   CloseHardwareBuffer(hardware_buffer);
-  // IMPORTANT: We only close the old image after egl_image_ stops referencing
-  // it.
+  // IMPORTANT: We have just received a new frame to display so close the
+  // previous Java Image so that it is recycled and used for a future frame.
   CloseImage(old_android_image);
-  return true;
+
+  if (!egl_image_.is_valid()) {
+    return;
+  }
+
+  dl_image_ = CreateDlImage(context, bounds);
+  AddImage(dl_image_, key);
+}
+
+void ImageExternalTextureGL::Detach() {
+  egl_image_.reset();
 }
 
 impeller::UniqueEGLImageKHR ImageExternalTextureGL::CreateEGLImage(
@@ -121,17 +142,6 @@ void ImageExternalTextureGLSkia::Detach() {
   texture_.reset();
 }
 
-void ImageExternalTextureGLSkia::ProcessFrame(PaintContext& context,
-                                              const SkRect& bounds) {
-  const bool swapped = MaybeSwapImages();
-  if (!swapped && !egl_image_.is_valid()) {
-    // Nothing to do.
-    return;
-  }
-  BindImageToTexture(egl_image_, texture_.get().texture_name);
-  dl_image_ = CreateDlImage(context, bounds);
-}
-
 void ImageExternalTextureGLSkia::BindImageToTexture(
     const impeller::UniqueEGLImageKHR& image,
     GLuint tex) {
@@ -146,6 +156,7 @@ void ImageExternalTextureGLSkia::BindImageToTexture(
 sk_sp<flutter::DlImage> ImageExternalTextureGLSkia::CreateDlImage(
     PaintContext& context,
     const SkRect& bounds) {
+  BindImageToTexture(egl_image_, texture_.get().texture_name);
   GrGLTextureInfo textureInfo = {GL_TEXTURE_EXTERNAL_OES,
                                  texture_.get().texture_name, GL_RGBA8_OES};
   auto backendTexture =
@@ -171,16 +182,6 @@ void ImageExternalTextureGLImpeller::Attach(PaintContext& context) {
   }
 }
 
-void ImageExternalTextureGLImpeller::ProcessFrame(PaintContext& context,
-                                                  const SkRect& bounds) {
-  const bool swapped = MaybeSwapImages();
-  if (!swapped && !egl_image_.is_valid()) {
-    // Nothing to do.
-    return;
-  }
-  dl_image_ = CreateDlImage(context, bounds);
-}
-
 sk_sp<flutter::DlImage> ImageExternalTextureGLImpeller::CreateDlImage(
     PaintContext& context,
     const SkRect& bounds) {

shell/platform/android/image_external_texture_gl.h

@@ -30,10 +30,11 @@ class ImageExternalTextureGL : public ImageExternalTexture {
  protected:
   void Attach(PaintContext& context) override;
   void Detach() override;
+  void ProcessFrame(PaintContext& context, const SkRect& bounds) override;
+
+  virtual sk_sp<flutter::DlImage> CreateDlImage(PaintContext& context,
+                                                const SkRect& bounds) = 0;
 
-  // Returns true if a new image was acquired and android_image_ and egl_image_
-  // were updated.
-  bool MaybeSwapImages();
   impeller::UniqueEGLImageKHR CreateEGLImage(AHardwareBuffer* buffer);
 
   fml::jni::ScopedJavaGlobalRef<jobject> android_image_;
@@ -53,11 +54,10 @@ class ImageExternalTextureGLSkia : public ImageExternalTextureGL {
  private:
   void Attach(PaintContext& context) override;
   void Detach() override;
-  void ProcessFrame(PaintContext& context, const SkRect& bounds) override;
 
   void BindImageToTexture(const impeller::UniqueEGLImageKHR& image, GLuint tex);
   sk_sp<flutter::DlImage> CreateDlImage(PaintContext& context,
-                                        const SkRect& bounds);
+                                        const SkRect& bounds) override;
 
   impeller::UniqueGLTexture texture_;
 
@@ -75,11 +75,10 @@ class ImageExternalTextureGLImpeller : public ImageExternalTextureGL {
 
  private:
   void Attach(PaintContext& context) override;
-  void ProcessFrame(PaintContext& context, const SkRect& bounds) override;
   void Detach() override;
 
   sk_sp<flutter::DlImage> CreateDlImage(PaintContext& context,
-                                        const SkRect& bounds);
+                                        const SkRect& bounds) override;
 
   const std::shared_ptr<impeller::ContextGLES> impeller_context_;
 

shell/platform/android/image_external_texture_vk.cc

@@ -1,5 +1,6 @@
 
 #include "flutter/shell/platform/android/image_external_texture_vk.h"
+#include <cstdint>
 
 #include "flutter/impeller/core/formats.h"
 #include "flutter/impeller/core/texture_descriptor.h"
@@ -45,6 +46,17 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context,
   AHardwareBuffer_Desc hb_desc = {};
   flutter::NDKHelpers::AHardwareBuffer_describe(latest_hardware_buffer,
                                                 &hb_desc);
+  auto key = flutter::NDKHelpers::AHardwareBuffer_getId(latest_hardware_buffer);
+  auto existing_image = FindImage(key);
+  if (existing_image != nullptr) {
+    dl_image_ = existing_image;
+
+    CloseHardwareBuffer(hardware_buffer);
+    // IMPORTANT: We have just received a new frame to display so close the
+    // previous Java Image so that it is recycled and used for a future frame.
+    CloseImage(old_android_image);
+    return;
+  }
 
   impeller::TextureDescriptor desc;
   desc.storage_mode = impeller::StorageMode::kDevicePrivate;
@@ -88,9 +100,10 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context,
   }
 
   dl_image_ = impeller::DlImageImpeller::Make(texture);
+  AddImage(dl_image_, key);
   CloseHardwareBuffer(hardware_buffer);
-  // IMPORTANT: We only close the old image after texture stops referencing
-  // it.
+  // IMPORTANT: We have just received a new frame to display so close the
+  // previous Java Image so that it is recycled and used for a future frame.
   CloseImage(old_android_image);
 }
 

shell/platform/android/image_external_texture_vk.h

@@ -5,6 +5,8 @@
 #ifndef FLUTTER_SHELL_PLATFORM_ANDROID_IMAGE_EXTERNAL_TEXTURE_VK_H_
 #define FLUTTER_SHELL_PLATFORM_ANDROID_IMAGE_EXTERNAL_TEXTURE_VK_H_
 
+#include <cstdint>
+#include <utility>
 #include "flutter/shell/platform/android/image_external_texture.h"
 
 #include "flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h"

shell/platform/android/ndk_helpers.cc

@@ -22,6 +22,9 @@ typedef void (*fp_AHardwareBuffer_acquire)(AHardwareBuffer* buffer);
 typedef void (*fp_AHardwareBuffer_release)(AHardwareBuffer* buffer);
 typedef void (*fp_AHardwareBuffer_describe)(AHardwareBuffer* buffer,
                                             AHardwareBuffer_Desc* desc);
+typedef void (*fp_AHardwareBuffer_getId)(AHardwareBuffer* buffer,
+                                         uint64_t* outId);
+
 typedef EGLClientBuffer (*fp_eglGetNativeClientBufferANDROID)(
     AHardwareBuffer* buffer);
 
@@ -32,6 +35,8 @@ void (*_AHardwareBuffer_acquire)(AHardwareBuffer* buffer) = nullptr;
 void (*_AHardwareBuffer_release)(AHardwareBuffer* buffer) = nullptr;
 void (*_AHardwareBuffer_describe)(AHardwareBuffer* buffer,
                                   AHardwareBuffer_Desc* desc) = nullptr;
+void (*_AHardwareBuffer_getId)(AHardwareBuffer* buffer,
+                               uint64_t* outId) = nullptr;
 EGLClientBuffer (*_eglGetNativeClientBufferANDROID)(AHardwareBuffer* buffer) =
     nullptr;
 
@@ -61,6 +66,10 @@ void InitOnceCallback() {
                                  ->ResolveFunction<fp_AHardwareBuffer_release>(
                                      "AHardwareBuffer_release")
                                  .value_or(nullptr);
+  _AHardwareBuffer_getId =
+      android
+          ->ResolveFunction<fp_AHardwareBuffer_getId>("AHardwareBuffer_getId")
+          .value_or(nullptr);
   _AHardwareBuffer_describe =
       android
           ->ResolveFunction<fp_AHardwareBuffer_describe>(
@@ -107,6 +116,14 @@ void NDKHelpers::AHardwareBuffer_describe(AHardwareBuffer* buffer,
   _AHardwareBuffer_describe(buffer, desc);
 }
 
+uint64_t NDKHelpers::AHardwareBuffer_getId(AHardwareBuffer* buffer) {
+  NDKHelpers::Init();
+  FML_CHECK(_AHardwareBuffer_getId != nullptr);
+  uint64_t outId;
+  _AHardwareBuffer_getId(buffer, &outId);
+  return outId;
+}
+
 EGLClientBuffer NDKHelpers::eglGetNativeClientBufferANDROID(
     AHardwareBuffer* buffer) {
   NDKHelpers::Init();

shell/platform/android/ndk_helpers.h

@@ -27,6 +27,7 @@ class NDKHelpers {
   static void AHardwareBuffer_release(AHardwareBuffer* buffer);
   static void AHardwareBuffer_describe(AHardwareBuffer* buffer,
                                        AHardwareBuffer_Desc* desc);
+  static uint64_t AHardwareBuffer_getId(AHardwareBuffer* buffer);
   static EGLClientBuffer eglGetNativeClientBufferANDROID(
       AHardwareBuffer* buffer);