diff --git a/package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm b/package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm index b3134ee5e8..91f552f4bb 100644 --- a/package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm +++ b/package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm @@ -1,11 +1,12 @@ -#include "RNSkiOSPlatformContext.h" +#import "RNSkiOSPlatformContext.h" #import #import #include #include -#include "SkiaMetalSurfaceFactory.h" +#import "SkiaCVPixelBufferUtils.h" +#import "SkiaMetalSurfaceFactory.h" #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdocumentation" @@ -69,63 +70,93 @@ } uint64_t RNSkiOSPlatformContext::makePlatformBuffer(sk_sp image) { + // 0. If Image is not in BGRA, convert to BGRA as only BGRA is supported. + if (image->colorType() != kBGRA_8888_SkColorType) { + // on iOS, 32_BGRA is the only supported RGB format for CVPixelBuffers. + image = image->makeColorTypeAndColorSpace( + ThreadContextHolder::ThreadSkiaMetalContext.skContext.get(), + kBGRA_8888_SkColorType, SkColorSpace::MakeSRGB()); + if (image == nullptr) { + throw std::runtime_error( + "Failed to convert image to BGRA_8888 colortype! Only BGRA_8888 " + "PlatformBuffers are supported."); + } + } + + // 1. Get image info auto bytesPerPixel = image->imageInfo().bytesPerPixel(); int bytesPerRow = image->width() * bytesPerPixel; auto buf = SkData::MakeUninitialized(image->width() * image->height() * bytesPerPixel); SkImageInfo info = SkImageInfo::Make(image->width(), image->height(), image->colorType(), image->alphaType()); + // 2. Copy pixels into our buffer image->readPixels(nullptr, info, const_cast(buf->data()), bytesPerRow, 0, 0); - auto pixelData = const_cast(buf->data()); - // Create a CVPixelBuffer from the raw pixel data - CVPixelBufferRef pixelBuffer = nullptr; - // OSType pixelFormatType = MapSkColorTypeToOSType(image->colorType()); - - // You will need to fill in the details for creating the pixel buffer - // CVPixelBufferCreateWithBytes or CVPixelBufferCreateWithPlanarBytes - // Create the CVPixelBuffer with the image data - void *context = static_cast( - new sk_sp(buf)); // Create a copy for the context - CVReturn r = CVPixelBufferCreateWithBytes( - nullptr, // allocator - image->width(), image->height(), kCVPixelFormatType_32BGRA, - pixelData, // pixel data - bytesPerRow, // bytes per row - [](void *releaseRefCon, const void *baseAddress) { // release callback - auto buf = static_cast *>(releaseRefCon); - buf->reset(); // This effectively calls unref on the SkData object - delete buf; // Cleanup the dynamically allocated context - }, - context, // release callback context - nullptr, // pixel buffer attributes - &pixelBuffer // the newly created pixel buffer - ); - - if (r != kCVReturnSuccess) { - return 0; // or handle error appropriately + // 3. Create an IOSurface (GPU + CPU memory) + CFMutableDictionaryRef dict = CFDictionaryCreateMutable( + kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + int width = image->width(); + int height = image->height(); + int pitch = width * bytesPerPixel; + int size = width * height * bytesPerPixel; + OSType pixelFormat = kCVPixelFormatType_32BGRA; + CFDictionarySetValue( + dict, kIOSurfaceBytesPerRow, + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pitch)); + CFDictionarySetValue( + dict, kIOSurfaceBytesPerElement, + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &bytesPerPixel)); + CFDictionarySetValue( + dict, kIOSurfaceWidth, + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width)); + CFDictionarySetValue( + dict, kIOSurfaceHeight, + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height)); + CFDictionarySetValue( + dict, kIOSurfacePixelFormat, + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat)); + CFDictionarySetValue( + dict, kIOSurfaceAllocSize, + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &size)); + IOSurfaceRef surface = IOSurfaceCreate(dict); + if (surface == nil) { + throw std::runtime_error("Failed to create " + std::to_string(width) + "x" + + std::to_string(height) + " IOSurface!"); } - // Wrap the CVPixelBuffer in a CMSampleBuffer - CMSampleBufferRef sampleBuffer = nullptr; + // 4. Copy over the memory from the pixels into the IOSurface + IOSurfaceLock(surface, 0, nil); + void *base = IOSurfaceGetBaseAddress(surface); + memcpy(base, buf->data(), buf->size()); + IOSurfaceUnlock(surface, 0, nil); + // 5. Create a CVPixelBuffer from the IOSurface + CVPixelBufferRef pixelBuffer = nullptr; + CVReturn result = + CVPixelBufferCreateWithIOSurface(nil, surface, nil, &pixelBuffer); + if (result != kCVReturnSuccess) { + throw std::runtime_error( + "Failed to create CVPixelBuffer from SkImage! Return value: " + + std::to_string(result)); + } + + // 6. Create CMSampleBuffer base information CMFormatDescriptionRef formatDescription = nullptr; CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &formatDescription); - - // Assuming no specific timing is required, we initialize the timing info to - // zero. CMSampleTimingInfo timingInfo = {0}; - timingInfo.duration = kCMTimeInvalid; // Indicate an unknown duration. - timingInfo.presentationTimeStamp = kCMTimeZero; // Start at time zero. - timingInfo.decodeTimeStamp = kCMTimeInvalid; // No specific decode time. + timingInfo.duration = kCMTimeInvalid; + timingInfo.presentationTimeStamp = kCMTimeZero; + timingInfo.decodeTimeStamp = kCMTimeInvalid; - // Create the sample buffer. + // 7. Wrap the CVPixelBuffer in a CMSampleBuffer + CMSampleBufferRef sampleBuffer = nullptr; OSStatus status = CMSampleBufferCreateReadyWithImageBuffer( kCFAllocatorDefault, pixelBuffer, formatDescription, &timingInfo, &sampleBuffer); - if (status != noErr) { if (formatDescription) { CFRelease(formatDescription); @@ -133,10 +164,12 @@ if (pixelBuffer) { CFRelease(pixelBuffer); } - return 0; + throw std::runtime_error( + "Failed to wrap CVPixelBuffer in CMSampleBuffer! Return value: " + + std::to_string(status)); } - // Return sampleBuffer casted to uint64_t + // 8. Return CMsampleBuffer casted to uint64_t return reinterpret_cast(sampleBuffer); } @@ -152,32 +185,7 @@ sk_sp RNSkiOSPlatformContext::makeImageFromPlatformBuffer(void *buffer) { CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)buffer; - // DO the CPU transfer (debugging only) - // Step 1: Extract the CVPixelBufferRef from the CMSampleBufferRef - CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); - - // Step 2: Lock the pixel buffer to access the raw pixel data - CVPixelBufferLockBaseAddress(pixelBuffer, 0); - - // Step 3: Get information about the image - void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); - size_t width = CVPixelBufferGetWidth(pixelBuffer); - size_t height = CVPixelBufferGetHeight(pixelBuffer); - size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); - - // Assuming the pixel format is 32BGRA, which is common for iOS video frames. - // You might need to adjust this based on the actual pixel format. - SkImageInfo info = SkImageInfo::Make(width, height, kRGBA_8888_SkColorType, - kUnpremul_SkAlphaType); - - // Step 4: Create an SkImage from the pixel buffer - sk_sp data = - SkData::MakeWithoutCopy(baseAddress, height * bytesPerRow); - sk_sp image = SkImages::RasterFromData(info, data, bytesPerRow); - auto texture = SkiaMetalSurfaceFactory::makeTextureFromImage(image); - // Step 5: Unlock the pixel buffer - CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); - return texture; + return SkiaMetalSurfaceFactory::makeTextureFromCMSampleBuffer(sampleBuffer); } sk_sp RNSkiOSPlatformContext::createFontMgr() { diff --git a/package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.h b/package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.h new file mode 100644 index 0000000000..7610d8ca94 --- /dev/null +++ b/package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.h @@ -0,0 +1,52 @@ +// +// SkiaCVPixelBufferUtils.h +// react-native-skia +// +// Created by Marc Rousavy on 10.04.24. +// + +#pragma once +#import +#import +#import + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdocumentation" +#import "include/core/SkColorSpace.h" +#import +#pragma clang diagnostic pop + +class SkiaCVPixelBufferUtils { +public: + enum class CVPixelBufferBaseFormat { rgb }; + + /** + Get the base format (currently only RGB) of the PixelBuffer. + Depending on the base-format, different methods have to be used to create + Skia buffers. + */ + static CVPixelBufferBaseFormat + getCVPixelBufferBaseFormat(CVPixelBufferRef pixelBuffer); + + class RGB { + public: + /** + Gets the Skia Color Type of the RGB pixel-buffer. + */ + static SkColorType getCVPixelBufferColorType(CVPixelBufferRef pixelBuffer); + /** + Gets a GPU-backed Skia Texture for the given RGB CVPixelBuffer. + */ + static GrBackendTexture + getSkiaTextureForCVPixelBuffer(CVPixelBufferRef pixelBuffer); + }; + +private: + static CVMetalTextureCacheRef getTextureCache(); + static GrBackendTexture + getSkiaTextureForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer, + size_t planeIndex); + static MTLPixelFormat + getMTLPixelFormatForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer, + size_t planeIndex); +}; diff --git a/package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.mm b/package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.mm new file mode 100644 index 0000000000..15e7309e72 --- /dev/null +++ b/package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.mm @@ -0,0 +1,149 @@ +// +// SkiaCVPixelBufferUtils.mm +// react-native-skia +// +// Created by Marc Rousavy on 10.04.24. +// + +#import "SkiaCVPixelBufferUtils.h" +#import "SkiaMetalSurfaceFactory.h" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdocumentation" +#import "include/core/SkColorSpace.h" +#import +#pragma clang diagnostic pop + +#include +#if TARGET_RT_BIG_ENDIAN +#define FourCC2Str(fourcc) \ + (const char[]) { \ + *((char *)&fourcc), *(((char *)&fourcc) + 1), *(((char *)&fourcc) + 2), \ + *(((char *)&fourcc) + 3), 0 \ + } +#else +#define FourCC2Str(fourcc) \ + (const char[]) { \ + *(((char *)&fourcc) + 3), *(((char *)&fourcc) + 2), \ + *(((char *)&fourcc) + 1), *(((char *)&fourcc) + 0), 0 \ + } +#endif + +// pragma MARK: Base + +SkiaCVPixelBufferUtils::CVPixelBufferBaseFormat +SkiaCVPixelBufferUtils::getCVPixelBufferBaseFormat( + CVPixelBufferRef pixelBuffer) { + OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer); + + switch (format) { + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_32RGBA: + return CVPixelBufferBaseFormat::rgb; + default: + [[unlikely]] throw std::runtime_error( + "CVPixelBuffer has unsupported pixel-format! " + + std::string(FourCC2Str(format))); + } +} + +// pragma MARK: RGB + +SkColorType SkiaCVPixelBufferUtils::RGB::getCVPixelBufferColorType( + CVPixelBufferRef pixelBuffer) { + OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer); + + switch (format) { + case kCVPixelFormatType_32BGRA: + [[likely]] return kBGRA_8888_SkColorType; + case kCVPixelFormatType_32RGBA: + return kRGBA_8888_SkColorType; + // This can be extended with branches for specific RGB formats if new Apple + // uses new formats. + default: + [[unlikely]] throw std::runtime_error( + "CVPixelBuffer has unknown RGB format! " + + std::string(FourCC2Str(format))); + } +} + +GrBackendTexture SkiaCVPixelBufferUtils::RGB::getSkiaTextureForCVPixelBuffer( + CVPixelBufferRef pixelBuffer) { + return getSkiaTextureForCVPixelBufferPlane(pixelBuffer, /* planeIndex */ 0); +} + +// pragma MARK: CVPixelBuffer -> Skia Texture + +GrBackendTexture SkiaCVPixelBufferUtils::getSkiaTextureForCVPixelBufferPlane( + CVPixelBufferRef pixelBuffer, size_t planeIndex) { + // 1. Get cache + CVMetalTextureCacheRef textureCache = getTextureCache(); + + // 2. Get MetalTexture from CMSampleBuffer + CVMetalTextureRef textureHolder; + size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex); + size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex); + MTLPixelFormat pixelFormat = + getMTLPixelFormatForCVPixelBufferPlane(pixelBuffer, planeIndex); + CVReturn result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, + height, planeIndex, &textureHolder); + if (result != kCVReturnSuccess) [[unlikely]] { + throw std::runtime_error( + "Failed to create Metal Texture from CMSampleBuffer! Result: " + + std::to_string(result)); + } + + // 2. Unwrap the underlying MTLTexture + id mtlTexture = CVMetalTextureGetTexture(textureHolder); + if (mtlTexture == nil) [[unlikely]] { + throw std::runtime_error( + "Failed to get MTLTexture from CVMetalTextureRef!"); + } + + // 3. Wrap MTLTexture in Skia's GrBackendTexture + GrMtlTextureInfo textureInfo; + textureInfo.fTexture.retain((__bridge void *)mtlTexture); + GrBackendTexture texture = + GrBackendTexture((int)mtlTexture.width, (int)mtlTexture.height, + skgpu::Mipmapped::kNo, textureInfo); + CFRelease(textureHolder); + return texture; +} + +// pragma MARK: getTextureCache() + +CVMetalTextureCacheRef SkiaCVPixelBufferUtils::getTextureCache() { + static thread_local CVMetalTextureCacheRef textureCache = nil; + if (textureCache == nil) { + // Create a new Texture Cache + auto result = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, + MTLCreateSystemDefaultDevice(), nil, + &textureCache); + if (result != kCVReturnSuccess || textureCache == nil) { + throw std::runtime_error("Failed to create Metal Texture Cache!"); + } + } + return textureCache; +} + +// pragma MARK: Get CVPixelBuffer MTLPixelFormat + +MTLPixelFormat SkiaCVPixelBufferUtils::getMTLPixelFormatForCVPixelBufferPlane( + CVPixelBufferRef pixelBuffer, size_t planeIndex) { + size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex); + size_t bytesPerRow = + CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, planeIndex); + double bytesPerPixel = round(static_cast(bytesPerRow) / width); + if (bytesPerPixel == 1) { + return MTLPixelFormatR8Unorm; + } else if (bytesPerPixel == 2) { + return MTLPixelFormatRG8Unorm; + } else if (bytesPerPixel == 4) { + return MTLPixelFormatBGRA8Unorm; + } else [[unlikely]] { + throw std::runtime_error("Invalid bytes per row! Expected 1 (R), 2 (RG) or " + "4 (RGBA), but received " + + std::to_string(bytesPerPixel)); + } +} diff --git a/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.h b/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.h index 8d9ac40f8e..4b98b21c31 100644 --- a/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.h +++ b/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.h @@ -26,7 +26,8 @@ class SkiaMetalSurfaceFactory { int height); static sk_sp makeOffscreenSurface(int width, int height); - static sk_sp makeTextureFromImage(sk_sp image); + static sk_sp + makeTextureFromCMSampleBuffer(CMSampleBufferRef sampleBuffer); private: static id device; diff --git a/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.mm b/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.mm index 0e95d2c1bf..00e12e97f8 100644 --- a/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.mm +++ b/package/ios/RNSkia-iOS/SkiaMetalSurfaceFactory.mm @@ -1,6 +1,7 @@ #import "RNSkLog.h" -#include "SkiaMetalSurfaceFactory.h" +#import "SkiaCVPixelBufferUtils.h" +#import "SkiaMetalSurfaceFactory.h" #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdocumentation" @@ -106,12 +107,39 @@ return surface; } -sk_sp -SkiaMetalSurfaceFactory::makeTextureFromImage(sk_sp image) { +sk_sp SkiaMetalSurfaceFactory::makeTextureFromCMSampleBuffer( + CMSampleBufferRef sampleBuffer) { if (!SkiaMetalSurfaceFactory::createSkiaDirectContextIfNecessary( - &ThreadContextHolder::ThreadSkiaMetalContext)) { + &ThreadContextHolder::ThreadSkiaMetalContext)) [[unlikely]] { throw std::runtime_error("Failed to create Skia Context for this Thread!"); } - return SkImages::TextureFromImage( - ThreadContextHolder::ThreadSkiaMetalContext.skContext.get(), image); + const SkiaMetalContext &context = ThreadContextHolder::ThreadSkiaMetalContext; + + if (!CMSampleBufferIsValid(sampleBuffer)) [[unlikely]] { + throw std::runtime_error("The given CMSampleBuffer is not valid!"); + } + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + SkiaCVPixelBufferUtils::CVPixelBufferBaseFormat format = + SkiaCVPixelBufferUtils::getCVPixelBufferBaseFormat(pixelBuffer); + switch (format) { + case SkiaCVPixelBufferUtils::CVPixelBufferBaseFormat::rgb: { + // CVPixelBuffer is in any RGB format. + SkColorType colorType = + SkiaCVPixelBufferUtils::RGB::getCVPixelBufferColorType(pixelBuffer); + GrBackendTexture texture = + SkiaCVPixelBufferUtils::RGB::getSkiaTextureForCVPixelBuffer( + pixelBuffer); + return SkImages::AdoptTextureFrom(context.skContext.get(), texture, + kTopLeft_GrSurfaceOrigin, colorType, + kOpaque_SkAlphaType); + } + default: + [[unlikely]] { + throw std::runtime_error("Failed to convert PlatformBuffer to SkImage - " + "PlatformBuffer has unsupported PixelFormat! " + + std::to_string(static_cast(format))); + } + } }