feat: Use Metal for creating PlatformBuffers

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm

@@ -1,11 +1,12 @@
-#include "RNSkiOSPlatformContext.h"
+#import "RNSkiOSPlatformContext.h"
 
 #import <CoreMedia/CMSampleBuffer.h>
 #import <React/RCTUtils.h>
 #include <thread>
 #include <utility>
 
-#include "SkiaMetalSurfaceFactory.h"
+#import "SkiaCVPixelBufferUtils.h"
+#import "SkiaMetalSurfaceFactory.h"
 
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdocumentation"
@@ -69,74 +70,106 @@
 }
 
 uint64_t RNSkiOSPlatformContext::makePlatformBuffer(sk_sp<SkImage> image) {
+  // 0. If Image is not in BGRA, convert to BGRA as only BGRA is supported.
+  if (image->colorType() != kBGRA_8888_SkColorType) {
+    // on iOS, 32_BGRA is the only supported RGB format for CVPixelBuffers.
+    image = image->makeColorTypeAndColorSpace(
+        ThreadContextHolder::ThreadSkiaMetalContext.skContext.get(),
+        kBGRA_8888_SkColorType, SkColorSpace::MakeSRGB());
+    if (image == nullptr) {
+      throw std::runtime_error(
+          "Failed to convert image to BGRA_8888 colortype! Only BGRA_8888 "
+          "PlatformBuffers are supported.");
+    }
+  }
+
+  // 1. Get image info
   auto bytesPerPixel = image->imageInfo().bytesPerPixel();
   int bytesPerRow = image->width() * bytesPerPixel;
   auto buf = SkData::MakeUninitialized(image->width() * image->height() *
                                        bytesPerPixel);
   SkImageInfo info = SkImageInfo::Make(image->width(), image->height(),
                                        image->colorType(), image->alphaType());
+  // 2. Copy pixels into our buffer
   image->readPixels(nullptr, info, const_cast<void *>(buf->data()), bytesPerRow,
                     0, 0);
-  auto pixelData = const_cast<void *>(buf->data());
 
-  // Create a CVPixelBuffer from the raw pixel data
-  CVPixelBufferRef pixelBuffer = nullptr;
-  // OSType pixelFormatType = MapSkColorTypeToOSType(image->colorType());
-
-  // You will need to fill in the details for creating the pixel buffer
-  // CVPixelBufferCreateWithBytes or CVPixelBufferCreateWithPlanarBytes
-  // Create the CVPixelBuffer with the image data
-  void *context = static_cast<void *>(
-      new sk_sp<SkData>(buf)); // Create a copy for the context
-  CVReturn r = CVPixelBufferCreateWithBytes(
-      nullptr, // allocator
-      image->width(), image->height(), kCVPixelFormatType_32BGRA,
-      pixelData,                                         // pixel data
-      bytesPerRow,                                       // bytes per row
-      [](void *releaseRefCon, const void *baseAddress) { // release callback
-        auto buf = static_cast<sk_sp<SkData> *>(releaseRefCon);
-        buf->reset(); // This effectively calls unref on the SkData object
-        delete buf;   // Cleanup the dynamically allocated context
-      },
-      context,     // release callback context
-      nullptr,     // pixel buffer attributes
-      &pixelBuffer // the newly created pixel buffer
-  );
-
-  if (r != kCVReturnSuccess) {
-    return 0; // or handle error appropriately
+  // 3. Create an IOSurface (GPU + CPU memory)
+  CFMutableDictionaryRef dict = CFDictionaryCreateMutable(
+      kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks,
+      &kCFTypeDictionaryValueCallBacks);
+  int width = image->width();
+  int height = image->height();
+  int pitch = width * bytesPerPixel;
+  int size = width * height * bytesPerPixel;
+  OSType pixelFormat = kCVPixelFormatType_32BGRA;
+  CFDictionarySetValue(
+      dict, kIOSurfaceBytesPerRow,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pitch));
+  CFDictionarySetValue(
+      dict, kIOSurfaceBytesPerElement,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &bytesPerPixel));
+  CFDictionarySetValue(
+      dict, kIOSurfaceWidth,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width));
+  CFDictionarySetValue(
+      dict, kIOSurfaceHeight,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height));
+  CFDictionarySetValue(
+      dict, kIOSurfacePixelFormat,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat));
+  CFDictionarySetValue(
+      dict, kIOSurfaceAllocSize,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &size));
+  IOSurfaceRef surface = IOSurfaceCreate(dict);
+  if (surface == nil) {
+    throw std::runtime_error("Failed to create " + std::to_string(width) + "x" +
+                             std::to_string(height) + " IOSurface!");
   }
 
-  // Wrap the CVPixelBuffer in a CMSampleBuffer
-  CMSampleBufferRef sampleBuffer = nullptr;
+  // 4. Copy over the memory from the pixels into the IOSurface
+  IOSurfaceLock(surface, 0, nil);
+  void *base = IOSurfaceGetBaseAddress(surface);
+  memcpy(base, buf->data(), buf->size());
+  IOSurfaceUnlock(surface, 0, nil);
 
+  // 5. Create a CVPixelBuffer from the IOSurface
+  CVPixelBufferRef pixelBuffer = nullptr;
+  CVReturn result =
+      CVPixelBufferCreateWithIOSurface(nil, surface, nil, &pixelBuffer);
+  if (result != kCVReturnSuccess) {
+    throw std::runtime_error(
+        "Failed to create CVPixelBuffer from SkImage! Return value: " +
+        std::to_string(result));
+  }
+
+  // 6. Create CMSampleBuffer base information
   CMFormatDescriptionRef formatDescription = nullptr;
   CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer,
                                                &formatDescription);
-
-  // Assuming no specific timing is required, we initialize the timing info to
-  // zero.
   CMSampleTimingInfo timingInfo = {0};
-  timingInfo.duration = kCMTimeInvalid; // Indicate an unknown duration.
-  timingInfo.presentationTimeStamp = kCMTimeZero; // Start at time zero.
-  timingInfo.decodeTimeStamp = kCMTimeInvalid;    // No specific decode time.
+  timingInfo.duration = kCMTimeInvalid;
+  timingInfo.presentationTimeStamp = kCMTimeZero;
+  timingInfo.decodeTimeStamp = kCMTimeInvalid;
 
-  // Create the sample buffer.
+  // 7. Wrap the CVPixelBuffer in a CMSampleBuffer
+  CMSampleBufferRef sampleBuffer = nullptr;
   OSStatus status = CMSampleBufferCreateReadyWithImageBuffer(
       kCFAllocatorDefault, pixelBuffer, formatDescription, &timingInfo,
       &sampleBuffer);
-
   if (status != noErr) {
     if (formatDescription) {
       CFRelease(formatDescription);
     }
     if (pixelBuffer) {
       CFRelease(pixelBuffer);
     }
-    return 0;
+    throw std::runtime_error(
+        "Failed to wrap CVPixelBuffer in CMSampleBuffer! Return value: " +
+        std::to_string(status));
   }
 
-  // Return sampleBuffer casted to uint64_t
+  // 8. Return CMsampleBuffer casted to uint64_t
   return reinterpret_cast<uint64_t>(sampleBuffer);
 }
 
@@ -152,32 +185,7 @@
 sk_sp<SkImage>
 RNSkiOSPlatformContext::makeImageFromPlatformBuffer(void *buffer) {
   CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)buffer;
-  // DO the CPU transfer (debugging only)
-  //  Step 1: Extract the CVPixelBufferRef from the CMSampleBufferRef
-  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
-
-  // Step 2: Lock the pixel buffer to access the raw pixel data
-  CVPixelBufferLockBaseAddress(pixelBuffer, 0);
-
-  // Step 3: Get information about the image
-  void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer);
-  size_t width = CVPixelBufferGetWidth(pixelBuffer);
-  size_t height = CVPixelBufferGetHeight(pixelBuffer);
-  size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
-
-  // Assuming the pixel format is 32BGRA, which is common for iOS video frames.
-  // You might need to adjust this based on the actual pixel format.
-  SkImageInfo info = SkImageInfo::Make(width, height, kRGBA_8888_SkColorType,
-                                       kUnpremul_SkAlphaType);
-
-  // Step 4: Create an SkImage from the pixel buffer
-  sk_sp<SkData> data =
-      SkData::MakeWithoutCopy(baseAddress, height * bytesPerRow);
-  sk_sp<SkImage> image = SkImages::RasterFromData(info, data, bytesPerRow);
-  auto texture = SkiaMetalSurfaceFactory::makeTextureFromImage(image);
-  // Step 5: Unlock the pixel buffer
-  CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
-  return texture;
+  return SkiaMetalSurfaceFactory::makeTextureFromCMSampleBuffer(sampleBuffer);
 }
 
 sk_sp<SkFontMgr> RNSkiOSPlatformContext::createFontMgr() {

package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.h

@@ -0,0 +1,52 @@
+//
+//  SkiaCVPixelBufferUtils.h
+//  react-native-skia
+//
+//  Created by Marc Rousavy on 10.04.24.
+//
+
+#pragma once
+#import <CoreMedia/CMSampleBuffer.h>
+#import <CoreVideo/CVMetalTextureCache.h>
+#import <MetalKit/MetalKit.h>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+#import "include/core/SkColorSpace.h"
+#import <include/gpu/GrBackendSurface.h>
+#pragma clang diagnostic pop
+
+class SkiaCVPixelBufferUtils {
+public:
+  enum class CVPixelBufferBaseFormat { rgb };
+
+  /**
+   Get the base format (currently only RGB) of the PixelBuffer.
+   Depending on the base-format, different methods have to be used to create
+   Skia buffers.
+   */
+  static CVPixelBufferBaseFormat
+  getCVPixelBufferBaseFormat(CVPixelBufferRef pixelBuffer);
+
+  class RGB {
+  public:
+    /**
+     Gets the Skia Color Type of the RGB pixel-buffer.
+     */
+    static SkColorType getCVPixelBufferColorType(CVPixelBufferRef pixelBuffer);
+    /**
+     Gets a GPU-backed Skia Texture for the given RGB CVPixelBuffer.
+     */
+    static GrBackendTexture
+    getSkiaTextureForCVPixelBuffer(CVPixelBufferRef pixelBuffer);
+  };
+
+private:
+  static CVMetalTextureCacheRef getTextureCache();
+  static GrBackendTexture
+  getSkiaTextureForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer,
+                                      size_t planeIndex);
+  static MTLPixelFormat
+  getMTLPixelFormatForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer,
+                                         size_t planeIndex);
+};