diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 7a06e1efa04bd..895c098e2d10a 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -189,6 +189,7 @@ FILE: ../../../flutter/fml/platform/android/scoped_java_ref.cc FILE: ../../../flutter/fml/platform/android/scoped_java_ref.h FILE: ../../../flutter/fml/platform/darwin/cf_utils.cc FILE: ../../../flutter/fml/platform/darwin/cf_utils.h +FILE: ../../../flutter/fml/platform/darwin/cf_utils_unittests.mm FILE: ../../../flutter/fml/platform/darwin/message_loop_darwin.h FILE: ../../../flutter/fml/platform/darwin/message_loop_darwin.mm FILE: ../../../flutter/fml/platform/darwin/paths_darwin.mm @@ -890,6 +891,8 @@ FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_software.h FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_software.mm FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.h FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.mm +FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_metal.h +FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_metal.mm FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.h FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.mm FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface.h diff --git a/flow/texture.cc b/flow/texture.cc index 15c93d360366e..c81314bf017fe 100644 --- a/flow/texture.cc +++ b/flow/texture.cc @@ -13,12 +13,19 @@ Texture::~Texture() = default; TextureRegistry::TextureRegistry() = default; void TextureRegistry::RegisterTexture(std::shared_ptr texture) { + if (!texture) { + return; + } mapping_[texture->Id()] = texture; } void TextureRegistry::UnregisterTexture(int64_t id) { - mapping_[id]->OnTextureUnregistered(); - mapping_.erase(id); + auto found = mapping_.find(id); + if (found == mapping_.end()) { + return; + } + found->second->OnTextureUnregistered(); + mapping_.erase(found); } void TextureRegistry::OnGrContextCreated() { diff --git a/fml/BUILD.gn b/fml/BUILD.gn index 3a0b77515c4d8..08d60aaad7b9f 100644 --- a/fml/BUILD.gn +++ b/fml/BUILD.gn @@ -252,7 +252,6 @@ executable("fml_unittests") { "message_loop_unittests.cc", "message_unittests.cc", "paths_unittests.cc", - "platform/darwin/string_range_sanitization_unittests.mm", "synchronization/count_down_latch_unittests.cc", "synchronization/semaphore_unittest.cc", "synchronization/sync_switch_unittest.cc", @@ -264,6 +263,13 @@ executable("fml_unittests") { "time/time_unittest.cc", ] + if (is_mac) { + sources += [ + "platform/darwin/cf_utils_unittests.mm", + "platform/darwin/string_range_sanitization_unittests.mm", + ] + } + deps = [ ":fml_fixtures", "//flutter/fml", diff --git a/fml/platform/darwin/cf_utils.h b/fml/platform/darwin/cf_utils.h index f6be04abaaa71..00ed82d1f4411 100644 --- a/fml/platform/darwin/cf_utils.h +++ b/fml/platform/darwin/cf_utils.h @@ -18,6 +18,21 @@ class CFRef { CFRef(T instance) : instance_(instance) {} + CFRef(const CFRef& other) : instance_(other.instance_) { + if (instance_) { + CFRetain(instance_); + } + } + + CFRef(CFRef&& other) : instance_(other.instance_) { + other.instance_ = nullptr; + } + + CFRef& operator=(CFRef&& other) { + Reset(other.Release()); + return *this; + } + ~CFRef() { if (instance_ != nullptr) { CFRelease(instance_); @@ -25,7 +40,7 @@ class CFRef { instance_ = nullptr; } - void Reset(T instance) { + void Reset(T instance = nullptr) { if (instance_ == instance) { return; } @@ -36,6 +51,12 @@ class CFRef { instance_ = instance; } + [[nodiscard]] T Release() { + auto instance = instance_; + instance_ = nullptr; + return instance; + } + operator T() const { return instance_; } operator bool() const { return instance_ != nullptr; } @@ -43,7 +64,7 @@ class CFRef { private: T instance_; - FML_DISALLOW_COPY_AND_ASSIGN(CFRef); + CFRef& operator=(const CFRef&) = delete; }; } // namespace fml diff --git a/fml/platform/darwin/cf_utils_unittests.mm b/fml/platform/darwin/cf_utils_unittests.mm new file mode 100644 index 0000000000000..9df1f5f6ddfd3 --- /dev/null +++ b/fml/platform/darwin/cf_utils_unittests.mm @@ -0,0 +1,63 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/fml/platform/darwin/cf_utils.h" +#include "flutter/testing/testing.h" + +namespace fml { +namespace testing { + +TEST(CFTest, CanCreateRefs) { + CFRef string(CFStringCreateMutable(kCFAllocatorDefault, 100u)); + // Cast + ASSERT_TRUE(static_cast(string)); + ASSERT_TRUE(string); + + const auto ref_count = CFGetRetainCount(string); + + // Copy & Reset + { + CFRef string2 = string; + ASSERT_TRUE(string2); + ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string)); + ASSERT_EQ(CFGetRetainCount(string2), CFGetRetainCount(string)); + + string2.Reset(); + ASSERT_FALSE(string2); + ASSERT_EQ(ref_count, CFGetRetainCount(string)); + } + + // Release + { + auto string3 = string; + ASSERT_TRUE(string3); + ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string)); + auto raw_string3 = string3.Release(); + ASSERT_FALSE(string3); + ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string)); + CFRelease(raw_string3); + ASSERT_EQ(ref_count, CFGetRetainCount(string)); + } + + // Move + { + auto string_source = string; + ASSERT_TRUE(string_source); + auto string_move = std::move(string_source); + ASSERT_FALSE(string_source); + ASSERT_EQ(ref_count + 1u, CFGetRetainCount(string)); + string_move.Reset(); + ASSERT_EQ(ref_count, CFGetRetainCount(string)); + } + + // Move assign. + { + auto string_move_assign = std::move(string); + ASSERT_FALSE(string); + ASSERT_EQ(ref_count, CFGetRetainCount(string_move_assign)); + } +} + +} // namespace testing +} // namespace fml diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn index 1671975c8d0e5..7af51b5aa230d 100644 --- a/shell/platform/darwin/ios/BUILD.gn +++ b/shell/platform/darwin/ios/BUILD.gn @@ -131,6 +131,8 @@ shared_library("create_flutter_framework_dylib") { sources += [ "ios_context_metal.h", "ios_context_metal.mm", + "ios_external_texture_metal.h", + "ios_external_texture_metal.mm", "ios_surface_metal.h", "ios_surface_metal.mm", ] diff --git a/shell/platform/darwin/ios/ios_context.h b/shell/platform/darwin/ios/ios_context.h index 6615821905807..37fccd4b3de2e 100644 --- a/shell/platform/darwin/ios/ios_context.h +++ b/shell/platform/darwin/ios/ios_context.h @@ -7,8 +7,10 @@ #include +#include "flutter/flow/texture.h" #include "flutter/fml/macros.h" #include "flutter/fml/platform/darwin/scoped_nsobject.h" +#include "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" #include "flutter/shell/platform/darwin/ios/rendering_api_selection.h" #include "third_party/skia/include/gpu/GrContext.h" @@ -119,6 +121,20 @@ class IOSContext { /// virtual bool ClearCurrent() = 0; + //---------------------------------------------------------------------------- + /// @brief Creates an external texture proxy of the appropriate client + /// rendering API. + /// + /// @param[in] texture_id The texture identifier + /// @param[in] texture The texture + /// + /// @return The texture proxy if the rendering backend supports embedder + /// provided external textures. + /// + virtual std::unique_ptr CreateExternalTexture( + int64_t texture_id, + fml::scoped_nsobject> texture) = 0; + protected: IOSContext(); diff --git a/shell/platform/darwin/ios/ios_context_gl.h b/shell/platform/darwin/ios/ios_context_gl.h index fd8885a42395a..56308831b954c 100644 --- a/shell/platform/darwin/ios/ios_context_gl.h +++ b/shell/platform/darwin/ios/ios_context_gl.h @@ -40,6 +40,11 @@ class IOSContextGL final : public IOSContext { // |IOSContext| bool ResourceMakeCurrent() override; + // |IOSContext| + std::unique_ptr CreateExternalTexture( + int64_t texture_id, + fml::scoped_nsobject> texture) override; + FML_DISALLOW_COPY_AND_ASSIGN(IOSContextGL); }; diff --git a/shell/platform/darwin/ios/ios_context_gl.mm b/shell/platform/darwin/ios/ios_context_gl.mm index 1396198338193..2a5a32bf8c5cb 100644 --- a/shell/platform/darwin/ios/ios_context_gl.mm +++ b/shell/platform/darwin/ios/ios_context_gl.mm @@ -8,6 +8,7 @@ #include "flutter/shell/common/shell_io_manager.h" #include "flutter/shell/gpu/gpu_surface_gl_delegate.h" +#include "flutter/shell/platform/darwin/ios/ios_external_texture_gl.h" namespace flutter { @@ -58,4 +59,11 @@ return [EAGLContext setCurrentContext:nil]; } +// |IOSContext| +std::unique_ptr IOSContextGL::CreateExternalTexture( + int64_t texture_id, + fml::scoped_nsobject> texture) { + return std::make_unique(texture_id, std::move(texture)); +} + } // namespace flutter diff --git a/shell/platform/darwin/ios/ios_context_metal.h b/shell/platform/darwin/ios/ios_context_metal.h index f98b11f72d9cc..194b3fc4a0084 100644 --- a/shell/platform/darwin/ios/ios_context_metal.h +++ b/shell/platform/darwin/ios/ios_context_metal.h @@ -8,6 +8,7 @@ #include #include "flutter/fml/macros.h" +#include "flutter/fml/platform/darwin/cf_utils.h" #include "flutter/fml/platform/darwin/scoped_nsobject.h" #include "flutter/shell/platform/darwin/ios/ios_context.h" #include "third_party/skia/include/gpu/GrContext.h" @@ -35,6 +36,7 @@ class IOSContextMetal final : public IOSContext { fml::scoped_nsprotocol> main_queue_; sk_sp main_context_; sk_sp resource_context_; + fml::CFRef texture_cache_; bool is_valid_ = false; // |IOSContext| @@ -49,6 +51,11 @@ class IOSContextMetal final : public IOSContext { // |IOSContext| bool ClearCurrent() override; + // |IOSContext| + std::unique_ptr CreateExternalTexture( + int64_t texture_id, + fml::scoped_nsobject> texture) override; + FML_DISALLOW_COPY_AND_ASSIGN(IOSContextMetal); }; diff --git a/shell/platform/darwin/ios/ios_context_metal.mm b/shell/platform/darwin/ios/ios_context_metal.mm index 8b9cbda33455b..eb16aab4a7008 100644 --- a/shell/platform/darwin/ios/ios_context_metal.mm +++ b/shell/platform/darwin/ios/ios_context_metal.mm @@ -5,20 +5,21 @@ #include "flutter/shell/platform/darwin/ios/ios_context_metal.h" #include "flutter/fml/logging.h" +#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" namespace flutter { IOSContextMetal::IOSContextMetal() { device_.reset([MTLCreateSystemDefaultDevice() retain]); if (!device_) { - FML_LOG(ERROR) << "Could not acquire Metal device."; + FML_DLOG(ERROR) << "Could not acquire Metal device."; return; } main_queue_.reset([device_ newCommandQueue]); if (!main_queue_) { - FML_LOG(ERROR) << "Could not create Metal command queue."; + FML_DLOG(ERROR) << "Could not create Metal command queue."; return; } @@ -30,10 +31,23 @@ resource_context_ = GrContext::MakeMetal([device_ retain], [main_queue_ retain]); if (!main_context_ || !resource_context_) { - FML_LOG(ERROR) << "Could not create Skia Metal contexts."; + FML_DLOG(ERROR) << "Could not create Skia Metal contexts."; return; } + CVMetalTextureCacheRef texture_cache_raw = NULL; + auto cv_return = CVMetalTextureCacheCreate(kCFAllocatorDefault, // allocator + NULL, // cache attributes (NULL default) + device_.get(), // metal device + NULL, // texture attributes (NULL default) + &texture_cache_raw // [out] cache + ); + if (cv_return != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture cache."; + return; + } + texture_cache_.Reset(texture_cache_raw); + is_valid_ = false; } @@ -83,4 +97,11 @@ return true; } +// |IOSContext| +std::unique_ptr IOSContextMetal::CreateExternalTexture( + int64_t texture_id, + fml::scoped_nsobject> texture) { + return std::make_unique(texture_id, texture_cache_, std::move(texture)); +} + } // namespace flutter diff --git a/shell/platform/darwin/ios/ios_context_software.h b/shell/platform/darwin/ios/ios_context_software.h index 558f39a637f97..0bd11c4538b43 100644 --- a/shell/platform/darwin/ios/ios_context_software.h +++ b/shell/platform/darwin/ios/ios_context_software.h @@ -29,6 +29,11 @@ class IOSContextSoftware final : public IOSContext { // |IOSContext| bool ClearCurrent() override; + // |IOSContext| + std::unique_ptr CreateExternalTexture( + int64_t texture_id, + fml::scoped_nsobject> texture) override; + private: FML_DISALLOW_COPY_AND_ASSIGN(IOSContextSoftware); }; diff --git a/shell/platform/darwin/ios/ios_context_software.mm b/shell/platform/darwin/ios/ios_context_software.mm index 8bb029d5792d3..ed11bd542404d 100644 --- a/shell/platform/darwin/ios/ios_context_software.mm +++ b/shell/platform/darwin/ios/ios_context_software.mm @@ -31,4 +31,18 @@ return false; } +// |IOSContext| +std::unique_ptr IOSContextSoftware::CreateExternalTexture( + int64_t texture_id, + fml::scoped_nsobject> texture) { + // Don't use FML for logging as it will contain engine specific details. This is a user facing + // message. + NSLog(@"Flutter: Attempted to composite external texture sources using the software backend. " + @"This backend is only used on simulators. This feature is only available on actual " + @"devices where OpenGL or Metal is used for rendering."); + + // Not supported in this backend. + return nullptr; +} + } // namespace flutter diff --git a/shell/platform/darwin/ios/ios_external_texture_gl.mm b/shell/platform/darwin/ios/ios_external_texture_gl.mm index 19cb324ca8c62..697992d1b7cd8 100644 --- a/shell/platform/darwin/ios/ios_external_texture_gl.mm +++ b/shell/platform/darwin/ios/ios_external_texture_gl.mm @@ -56,7 +56,7 @@ bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) { // Update texture if `texture_ref_` is reset to `nullptr` when GrContext - // is destroied or new frame is ready. + // is destroyed or new frame is ready. return (!freeze && new_frame_ready_) || !texture_ref_; } diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.h b/shell/platform/darwin/ios/ios_external_texture_metal.h new file mode 100644 index 0000000000000..82e4d6931d426 --- /dev/null +++ b/shell/platform/darwin/ios/ios_external_texture_metal.h @@ -0,0 +1,58 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_ +#define FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_ + +#include + +#import + +#include "flutter/flow/texture.h" +#include "flutter/fml/macros.h" +#include "flutter/fml/platform/darwin/cf_utils.h" +#include "flutter/fml/platform/darwin/scoped_nsobject.h" +#include "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" +#include "third_party/skia/include/core/SkImage.h" + +namespace flutter { + +class IOSExternalTextureMetal final : public Texture { + public: + IOSExternalTextureMetal(int64_t texture_id, + fml::CFRef texture_cache, + fml::scoped_nsobject> external_texture); + + // |Texture| + ~IOSExternalTextureMetal(); + + private: + fml::CFRef texture_cache_; + fml::scoped_nsobject> external_texture_; + std::atomic_bool texture_frame_available_; + sk_sp external_image_; + + // |Texture| + void Paint(SkCanvas& canvas, const SkRect& bounds, bool freeze, GrContext* context) override; + + // |Texture| + void OnGrContextCreated() override; + + // |Texture| + void OnGrContextDestroyed() override; + + // |Texture| + void MarkNewFrameAvailable() override; + + // |Texture| + void OnTextureUnregistered() override; + + sk_sp WrapExternalPixelBuffer(GrContext* context); + + FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_ diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.mm b/shell/platform/darwin/ios/ios_external_texture_metal.mm new file mode 100644 index 0000000000000..7ab55c94b131d --- /dev/null +++ b/shell/platform/darwin/ios/ios_external_texture_metal.mm @@ -0,0 +1,143 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" + +#include "flutter/fml/logging.h" +#include "third_party/skia/include/gpu/GrBackendSurface.h" +#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h" + +namespace flutter { + +IOSExternalTextureMetal::IOSExternalTextureMetal( + int64_t texture_id, + fml::CFRef texture_cache, + fml::scoped_nsobject> external_texture) + : Texture(texture_id), + texture_cache_(std::move(texture_cache)), + external_texture_(std::move(external_texture)) { + FML_DCHECK(texture_cache_); + FML_DCHECK(external_texture_); +} + +IOSExternalTextureMetal::~IOSExternalTextureMetal() = default; + +void IOSExternalTextureMetal::Paint(SkCanvas& canvas, + const SkRect& bounds, + bool freeze, + GrContext* context) { + if (!freeze && texture_frame_available_) { + external_image_ = nullptr; + } + + if (!external_image_) { + external_image_ = WrapExternalPixelBuffer(context); + texture_frame_available_ = false; + } + + if (external_image_) { + canvas.drawImageRect(external_image_, // image + external_image_->bounds(), // source rect + bounds, // destination rect + nullptr, // paint + SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint + ); + } +} + +sk_sp IOSExternalTextureMetal::WrapExternalPixelBuffer(GrContext* context) { + auto pixel_buffer = fml::CFRef([external_texture_ copyPixelBuffer]); + if (!pixel_buffer) { + return nullptr; + } + + auto texture_size = + SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer)); + + CVMetalTextureRef metal_texture_raw = NULL; + auto cv_return = + CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator + texture_cache_, // texture cache + pixel_buffer, // source image + NULL, // texture attributes + MTLPixelFormatBGRA8Unorm, // pixel format + texture_size.width(), // width + texture_size.height(), // height + 0u, // plane index + &metal_texture_raw // [out] texture + ); + + if (cv_return != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return; + return nullptr; + } + + fml::CFRef metal_texture(metal_texture_raw); + + GrMtlTextureInfo skia_texture_info; + skia_texture_info.fTexture = sk_cf_obj{ + [reinterpret_cast(CVMetalTextureGetTexture(metal_texture)) retain]}; + + GrBackendTexture skia_backend_texture(texture_size.width(), // width + texture_size.height(), // height + GrMipMapped ::kNo, // mip-mapped + skia_texture_info // texture info + ); + + struct ImageCaptures { + fml::CFRef buffer; + fml::CFRef texture; + }; + + auto captures = std::make_unique(); + captures->buffer = std::move(pixel_buffer); + captures->texture = std::move(metal_texture); + + SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) { + auto captures = reinterpret_cast(release_context); + delete captures; + }; + + auto image = SkImage::MakeFromTexture(context, // context + skia_backend_texture, // backend texture + kTopLeft_GrSurfaceOrigin, // origin + kBGRA_8888_SkColorType, // color type + kPremul_SkAlphaType, // alpha type + nullptr, // color space + release_proc, // release proc + captures.release() // release context + + ); + + if (!image) { + FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image."; + } + + return image; +} + +void IOSExternalTextureMetal::OnGrContextCreated() { + // External images in this backend have no thread affinity and are not tied to the context in any + // way. Instead, they are tied to the Metal device which is associated with the cache already and + // is consistent throughout the shell run. +} + +void IOSExternalTextureMetal::OnGrContextDestroyed() { + external_image_.reset(); + CVMetalTextureCacheFlush(texture_cache_, // cache + 0 // options (must be zero) + ); +} + +void IOSExternalTextureMetal::MarkNewFrameAvailable() { + texture_frame_available_ = true; +} + +void IOSExternalTextureMetal::OnTextureUnregistered() { + if ([external_texture_ respondsToSelector:@selector(onTextureUnregistered:)]) { + [external_texture_ onTextureUnregistered:external_texture_]; + } +} + +} // namespace flutter diff --git a/shell/platform/darwin/ios/platform_view_ios.mm b/shell/platform/darwin/ios/platform_view_ios.mm index 09d19b4e61699..53b70968b341d 100644 --- a/shell/platform/darwin/ios/platform_view_ios.mm +++ b/shell/platform/darwin/ios/platform_view_ios.mm @@ -4,18 +4,14 @@ #include "flutter/shell/platform/darwin/ios/platform_view_ios.h" -#import - #include #include "flutter/common/task_runners.h" #include "flutter/fml/synchronization/waitable_event.h" #include "flutter/fml/trace_event.h" #include "flutter/shell/common/shell_io_manager.h" -#include "flutter/shell/gpu/gpu_surface_gl_delegate.h" #include "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h" #include "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h" -#include "flutter/shell/platform/darwin/ios/ios_external_texture_gl.h" namespace flutter { @@ -92,7 +88,8 @@ new AccessibilityBridge(static_cast(owner_controller_.get().view), void PlatformViewIOS::RegisterExternalTexture(int64_t texture_id, NSObject* texture) { - RegisterTexture(std::make_shared(texture_id, texture)); + RegisterTexture(ios_context_->CreateExternalTexture( + texture_id, fml::scoped_nsobject>{[texture retain]})); } // |PlatformView|