[Android] Cache GPU resources using HardwareBuffer's id as key

impeller/renderer/backend/vulkan/context_vk.cc

@@ -156,7 +156,7 @@ void ContextVK::Setup(Settings settings) {
   // 1. The user has explicitly enabled it.
   // 2. We are in a combination of debug mode, and running on Android.
   // (It's possible 2 is overly conservative and we can simplify this)
-  auto enable_validation = settings.enable_validation;
+  auto enable_validation = false;  // settings.enable_validation;
 
 #if defined(FML_OS_ANDROID) && !defined(NDEBUG)
   enable_validation = true;

shell/platform/android/image_external_texture.cc

@@ -59,6 +59,50 @@ void ImageExternalTexture::OnGrContextCreated() {
   state_ = AttachmentState::kUninitialized;
 }
 
+sk_sp<flutter::DlImage> ImageExternalTexture::FindImage(uint64_t key) {
+  for (auto i = 0u; i < kImageReaderSwapchainSize; i++) {
+    if (images_[i].key == key) {
+      UpdateKey(key);
+      return images_[i].image;
+    }
+  }
+  return nullptr;
+}
+
+void ImageExternalTexture::UpdateKey(uint64_t key) {
+  if (keys_[0] == key) {
+    return;
+  }
+  auto i = 1u;
+  for (; i < kImageReaderSwapchainSize; i++) {
+    if (keys_[i] == key) {
+      break;
+    }
+  }
+  for (auto j = i; j > 0; j--) {
+    keys_[j] = keys_[j - 1];
+  }
+  keys_[0] = key;
+}
+
+void ImageExternalTexture::AddImage(const sk_sp<flutter::DlImage>& image,
+                                    uint64_t key) {
+  uint64_t lru_key = keys_[kImageReaderSwapchainSize - 1];
+  bool updated_image = false;
+  for (auto i = 0u; i < kImageReaderSwapchainSize; i++) {
+    if (images_[i].key == lru_key) {
+      updated_image = true;
+      images_[i] = LRUImage{.key = key, .image = image};
+      break;
+    }
+  }
+  if (!updated_image) {
+    keys_[0] = key;
+    images_[0] = LRUImage{.key = key, .image = image};
+  }
+  UpdateKey(key);
+}
+
 // Implementing flutter::ContextListener.
 void ImageExternalTexture::OnGrContextDestroyed() {
   if (state_ == AttachmentState::kAttached) {

shell/platform/android/image_external_texture.h

@@ -15,6 +15,8 @@
 
 namespace flutter {
 
+static constexpr size_t kImageReaderSwapchainSize = 3u;
+
 // External texture peered to a sequence of android.hardware.HardwareBuffers.
 //
 class ImageExternalTexture : public flutter::Texture {
@@ -49,6 +51,15 @@ class ImageExternalTexture : public flutter::Texture {
   virtual void Detach() = 0;
   virtual void ProcessFrame(PaintContext& context, const SkRect& bounds) = 0;
 
+  sk_sp<flutter::DlImage> FindImage(uint64_t key);
+  void UpdateKey(uint64_t key);
+  void AddImage(const sk_sp<flutter::DlImage>& image, uint64_t key);
+
+  struct LRUImage {
+    uint64_t key;
+    sk_sp<flutter::DlImage> image;
+  };
+
   JavaLocalRef AcquireLatestImage();
   void CloseImage(const fml::jni::JavaRef<jobject>& image);
   JavaLocalRef HardwareBufferFor(const fml::jni::JavaRef<jobject>& image);
@@ -62,6 +73,12 @@ class ImageExternalTexture : public flutter::Texture {
   enum class AttachmentState { kUninitialized, kAttached, kDetached };
   AttachmentState state_ = AttachmentState::kUninitialized;
   bool new_frame_ready_ = false;
+  std::array<LRUImage, kImageReaderSwapchainSize> images_ = {
+      LRUImage{.key = 0, .image = nullptr},
+      LRUImage{.key = 0, .image = nullptr},
+      LRUImage{.key = 0, .image = nullptr},
+  };
+  std::array<uint64_t, kImageReaderSwapchainSize> keys_ = {0, 0, 0};
 
   sk_sp<flutter::DlImage> dl_image_;
 

shell/platform/android/image_external_texture_gl.cc

@@ -51,27 +51,6 @@ void ImageExternalTextureGL::Detach() {
   egl_image_.reset();
 }
 
-bool ImageExternalTextureGL::MaybeSwapImages() {
-  JavaLocalRef image = AcquireLatestImage();
-  if (image.is_null()) {
-    return false;
-  }
-  // NOTE: In the following code it is important that old_android_image is
-  // not closed until after the update of egl_image_ otherwise the image might
-  // be closed before the old EGLImage referencing it has been deleted. After
-  // an image is closed the underlying HardwareBuffer may be recycled and used
-  // for a future frame.
-  JavaLocalRef old_android_image(android_image_);
-  android_image_.Reset(image);
-  JavaLocalRef hardware_buffer = HardwareBufferFor(image);
-  egl_image_ = CreateEGLImage(AHardwareBufferFor(hardware_buffer));
-  CloseHardwareBuffer(hardware_buffer);
-  // IMPORTANT: We only close the old image after egl_image_ stops referencing
-  // it.
-  CloseImage(old_android_image);
-  return true;
-}
-
 impeller::UniqueEGLImageKHR ImageExternalTextureGL::CreateEGLImage(
     AHardwareBuffer* hardware_buffer) {
   if (hardware_buffer == nullptr) {
@@ -123,13 +102,45 @@ void ImageExternalTextureGLSkia::Detach() {
 
 void ImageExternalTextureGLSkia::ProcessFrame(PaintContext& context,
                                               const SkRect& bounds) {
-  const bool swapped = MaybeSwapImages();
-  if (!swapped && !egl_image_.is_valid()) {
-    // Nothing to do.
+  JavaLocalRef image = AcquireLatestImage();
+  if (image.is_null()) {
+    return;
+  }
+
+  // NOTE: In the following code it is important that old_android_image is
+  // not closed until after the update of egl_image_ otherwise the image might
+  // be closed before the old EGLImage referencing it has been deleted. After
+  // an image is closed the underlying HardwareBuffer may be recycled and used
+  // for a future frame.
+  JavaLocalRef old_android_image(android_image_);
+  android_image_.Reset(image);
+  JavaLocalRef hardware_buffer = HardwareBufferFor(image);
+  AHardwareBuffer* latest_hardware_buffer = AHardwareBufferFor(hardware_buffer);
+  auto key = flutter::NDKHelpers::AHardwareBuffer_getId(latest_hardware_buffer);
+  auto existing_image = FindImage(key);
+  if (existing_image != nullptr) {
+    dl_image_ = existing_image;
+
+    CloseHardwareBuffer(hardware_buffer);
+    // IMPORTANT: We only close the old image after texture stops referencing
+    // it.
+    CloseImage(old_android_image);
     return;
   }
+
+  egl_image_ = CreateEGLImage(latest_hardware_buffer);
+  CloseHardwareBuffer(hardware_buffer);
+  // IMPORTANT: We only close the old image after egl_image_ stops referencing
+  // it.
+  CloseImage(old_android_image);
+
+  if (!egl_image_.is_valid()) {
+    return;
+  }
+
   BindImageToTexture(egl_image_, texture_.get().texture_name);
   dl_image_ = CreateDlImage(context, bounds);
+  AddImage(dl_image_, key);
 }
 
 void ImageExternalTextureGLSkia::BindImageToTexture(
@@ -173,12 +184,42 @@ void ImageExternalTextureGLImpeller::Attach(PaintContext& context) {
 
 void ImageExternalTextureGLImpeller::ProcessFrame(PaintContext& context,
                                                   const SkRect& bounds) {
-  const bool swapped = MaybeSwapImages();
-  if (!swapped && !egl_image_.is_valid()) {
-    // Nothing to do.
+  JavaLocalRef image = AcquireLatestImage();
+  if (image.is_null()) {
     return;
   }
+  // NOTE: In the following code it is important that old_android_image is
+  // not closed until after the update of egl_image_ otherwise the image might
+  // be closed before the old EGLImage referencing it has been deleted. After
+  // an image is closed the underlying HardwareBuffer may be recycled and used
+  // for a future frame.
+  JavaLocalRef old_android_image(android_image_);
+  android_image_.Reset(image);
+  JavaLocalRef hardware_buffer = HardwareBufferFor(image);
+  AHardwareBuffer* latest_hardware_buffer = AHardwareBufferFor(hardware_buffer);
+  auto key = flutter::NDKHelpers::AHardwareBuffer_getId(latest_hardware_buffer);
+  auto existing_image = FindImage(key);
+  if (existing_image != nullptr) {
+    dl_image_ = existing_image;
+
+    CloseHardwareBuffer(hardware_buffer);
+    // IMPORTANT: We only close the old image after texture stops referencing
+    // it.
+    CloseImage(old_android_image);
+    return;
+  }
+
+  egl_image_ = CreateEGLImage(latest_hardware_buffer);
+  CloseHardwareBuffer(hardware_buffer);
+  // IMPORTANT: We only close the old image after egl_image_ stops referencing
+  // it.
+  CloseImage(old_android_image);
+  if (!egl_image_.is_valid()) {
+    return;
+  }
+
   dl_image_ = CreateDlImage(context, bounds);
+  AddImage(dl_image_, key);
 }
 
 sk_sp<flutter::DlImage> ImageExternalTextureGLImpeller::CreateDlImage(

shell/platform/android/image_external_texture_gl.h

@@ -31,9 +31,6 @@ class ImageExternalTextureGL : public ImageExternalTexture {
   void Attach(PaintContext& context) override;
   void Detach() override;
 
-  // Returns true if a new image was acquired and android_image_ and egl_image_
-  // were updated.
-  bool MaybeSwapImages();
   impeller::UniqueEGLImageKHR CreateEGLImage(AHardwareBuffer* buffer);
 
   fml::jni::ScopedJavaGlobalRef<jobject> android_image_;

shell/platform/android/image_external_texture_vk.cc

@@ -1,5 +1,6 @@
 
 #include "flutter/shell/platform/android/image_external_texture_vk.h"
+#include <cstdint>
 
 #include "flutter/impeller/core/formats.h"
 #include "flutter/impeller/core/texture_descriptor.h"
@@ -45,6 +46,17 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context,
   AHardwareBuffer_Desc hb_desc = {};
   flutter::NDKHelpers::AHardwareBuffer_describe(latest_hardware_buffer,
                                                 &hb_desc);
+  auto key = flutter::NDKHelpers::AHardwareBuffer_getId(latest_hardware_buffer);
+  auto existing_image = FindImage(key);
+  if (existing_image != nullptr) {
+    dl_image_ = existing_image;
+
+    CloseHardwareBuffer(hardware_buffer);
+    // IMPORTANT: We only close the old image after texture stops referencing
+    // it.
+    CloseImage(old_android_image);
+    return;
+  }
 
   impeller::TextureDescriptor desc;
   desc.storage_mode = impeller::StorageMode::kDevicePrivate;
@@ -88,6 +100,7 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context,
   }
 
   dl_image_ = impeller::DlImageImpeller::Make(texture);
+  AddImage(dl_image_, key);
   CloseHardwareBuffer(hardware_buffer);
   // IMPORTANT: We only close the old image after texture stops referencing
   // it.

shell/platform/android/image_external_texture_vk.h

@@ -5,6 +5,8 @@
 #ifndef FLUTTER_SHELL_PLATFORM_ANDROID_IMAGE_EXTERNAL_TEXTURE_VK_H_
 #define FLUTTER_SHELL_PLATFORM_ANDROID_IMAGE_EXTERNAL_TEXTURE_VK_H_
 
+#include <cstdint>
+#include <utility>
 #include "flutter/shell/platform/android/image_external_texture.h"
 
 #include "flutter/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h"

shell/platform/android/ndk_helpers.cc

@@ -22,6 +22,9 @@ typedef void (*fp_AHardwareBuffer_acquire)(AHardwareBuffer* buffer);
 typedef void (*fp_AHardwareBuffer_release)(AHardwareBuffer* buffer);
 typedef void (*fp_AHardwareBuffer_describe)(AHardwareBuffer* buffer,
                                             AHardwareBuffer_Desc* desc);
+typedef void (*fp_AHardwareBuffer_getId)(AHardwareBuffer* buffer,
+                                         uint64_t* outId);
+
 typedef EGLClientBuffer (*fp_eglGetNativeClientBufferANDROID)(
     AHardwareBuffer* buffer);
 
@@ -32,6 +35,8 @@ void (*_AHardwareBuffer_acquire)(AHardwareBuffer* buffer) = nullptr;
 void (*_AHardwareBuffer_release)(AHardwareBuffer* buffer) = nullptr;
 void (*_AHardwareBuffer_describe)(AHardwareBuffer* buffer,
                                   AHardwareBuffer_Desc* desc) = nullptr;
+void (*_AHardwareBuffer_getId)(AHardwareBuffer* buffer,
+                               uint64_t* outId) = nullptr;
 EGLClientBuffer (*_eglGetNativeClientBufferANDROID)(AHardwareBuffer* buffer) =
     nullptr;
 
@@ -61,6 +66,10 @@ void InitOnceCallback() {
                                  ->ResolveFunction<fp_AHardwareBuffer_release>(
                                      "AHardwareBuffer_release")
                                  .value_or(nullptr);
+  _AHardwareBuffer_getId =
+      android
+          ->ResolveFunction<fp_AHardwareBuffer_getId>("AHardwareBuffer_getId")
+          .value_or(nullptr);
   _AHardwareBuffer_describe =
       android
           ->ResolveFunction<fp_AHardwareBuffer_describe>(
@@ -107,6 +116,14 @@ void NDKHelpers::AHardwareBuffer_describe(AHardwareBuffer* buffer,
   _AHardwareBuffer_describe(buffer, desc);
 }
 
+uint64_t NDKHelpers::AHardwareBuffer_getId(AHardwareBuffer* buffer) {
+  NDKHelpers::Init();
+  FML_CHECK(_AHardwareBuffer_getId != nullptr);
+  uint64_t outId;
+  _AHardwareBuffer_getId(buffer, &outId);
+  return outId;
+}
+
 EGLClientBuffer NDKHelpers::eglGetNativeClientBufferANDROID(
     AHardwareBuffer* buffer) {
   NDKHelpers::Init();

shell/platform/android/ndk_helpers.h

@@ -27,6 +27,7 @@ class NDKHelpers {
   static void AHardwareBuffer_release(AHardwareBuffer* buffer);
   static void AHardwareBuffer_describe(AHardwareBuffer* buffer,
                                        AHardwareBuffer_Desc* desc);
+  static uint64_t AHardwareBuffer_getId(AHardwareBuffer* buffer);
   static EGLClientBuffer eglGetNativeClientBufferANDROID(
       AHardwareBuffer* buffer);