diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 4e38dbec46..4776d3acbd 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -96,6 +96,7 @@ if (is_ios || is_mac) { "objc/base/RTCEncodedImage.h", "objc/base/RTCEncodedImage.m", "objc/base/RTCI420Buffer.h", + "objc/base/RTCNV12Buffer.h", "objc/base/RTCLogging.h", "objc/base/RTCLogging.mm", "objc/base/RTCMacros.h", @@ -596,6 +597,16 @@ if (is_ios || is_mac) { "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", "objc/components/video_frame_buffer/RTCCVPixelBuffer.mm", ] + # Stream-only: NV12 wrappers are built when the flag is enabled. + # Stream-only: enable gating macro for ObjC bridge policies. + # Stream-only: switch to NV12-enabled ObjC bridge implementation. + if (stream_enable_rendering_backend) { + sources += [ + "objc/api/video_frame_buffer/RTCNativeNV12Buffer+Private.h", + "objc/api/video_frame_buffer/RTCNativeNV12Buffer.h", + "objc/api/video_frame_buffer/RTCNativeNV12Buffer.mm", + ] + } deps = [ ":base_objc", "../rtc_base:logging", @@ -642,6 +653,8 @@ if (is_ios || is_mac) { sources += [ "objc/components/renderer/metal/RTCMTLVideoView.h", "objc/components/renderer/metal/RTCMTLVideoView.m", + "objc/components/renderer/metal/RTCVideoRenderingView.h", + "objc/components/renderer/metal/RTCVideoRenderingView.m", ] } if (is_mac) { @@ -657,6 +670,42 @@ if (is_ios || is_mac) { ] configs += [ "..:common_objc" ] public_configs = [ ":common_config_objc" ] + # Stream-only: add shared Metal backend as a separate target. + if (stream_enable_rendering_backend) { + defines = [ "RTC_STREAM_RENDERING_BACKEND" ] + deps += [ ":stream_shared_metal_objc" ] + } + } + + # Stream-only: iOS shared Metal implementation isolated here. + # XR (visionOS) is excluded until the shared backend is validated there. + if (stream_enable_rendering_backend && is_ios && + !(target_environment == "xrsimulator" || target_environment == "xrdevice")) { + rtc_library("stream_shared_metal_objc") { + visibility = [ "*" ] + sources = [ + "objc/components/renderer/metal/RTCSharedMetalRenderAdapter.h", + "objc/components/renderer/metal/RTCSharedMetalRenderAdapter.mm", + "objc/components/renderer/metal/RTCSharedMetalRenderingContext+Private.h", + "objc/components/renderer/metal/RTCSharedMetalRenderingContext.h", + "objc/components/renderer/metal/RTCSharedMetalRenderingContext.mm", + "objc/components/renderer/metal/RTCSharedMetalVideoView.h", + "objc/components/renderer/metal/RTCSharedMetalVideoView.mm", + ] + frameworks = [ + "CoreVideo.framework", + "Metal.framework", + "QuartzCore.framework", + "UIKit.framework", + ] + deps = [ + ":base_objc", + ":videoframebuffer_objc", + "../rtc_base:checks", + ] + configs += [ "..:common_objc" ] + public_configs = [ ":common_config_objc" ] + } } # TODO(bugs.webrtc.org/9627): Remove this target. @@ -1036,6 +1085,9 @@ if (is_ios || is_mac) { "environment_construction", ] configs += [ "..:no_global_constructors" ] + if (stream_enable_rendering_backend) { + defines = [ "RTC_STREAM_RENDERING_BACKEND" ] + } sources = [ "objc/api/peerconnection/RTCAudioDeviceModule.h", "objc/api/peerconnection/RTCAudioDeviceModule+Private.h", @@ -1528,6 +1580,13 @@ if (is_ios || is_mac) { "objc/components/audio/RTCAudioCustomProcessingDelegate.h", "objc/components/audio/RTCAudioProcessingConfig.h", ] + # Stream-only: export NV12 headers when the flag is enabled. + if (stream_enable_rendering_backend) { + common_objc_headers += [ + "objc/base/RTCNV12Buffer.h", + "objc/api/video_frame_buffer/RTCNativeNV12Buffer.h", + ] + } if (rtc_use_h265) { common_objc_headers += [ @@ -1542,6 +1601,7 @@ if (is_ios || is_mac) { common_objc_headers += [ "objc/helpers/RTCCameraPreviewView.h", "objc/components/renderer/metal/RTCMTLVideoView.h", + "objc/components/renderer/metal/RTCVideoRenderingView.h", ] } @@ -1692,6 +1752,7 @@ if (is_ios || is_mac) { "objc/components/capturer/RTCDesktopSource.h", "objc/components/capturer/RTCDesktopMediaList.h", "objc/components/renderer/metal/RTCMTLVideoView.h", + "objc/components/renderer/metal/RTCVideoRenderingView.h", "objc/components/renderer/metal/RTCMTLNSVideoView.h", "objc/components/renderer/opengl/RTCVideoViewShading.h", "objc/components/video_codec/RTCCodecSpecificInfoH264.h", @@ -1714,6 +1775,13 @@ if (is_ios || is_mac) { "objc/components/audio/RTCAudioCustomProcessingDelegate.h", "objc/components/audio/RTCAudioProcessingConfig.h", ] + # Stream-only: export NV12 headers when the flag is enabled. + if (stream_enable_rendering_backend) { + sources += [ + "objc/base/RTCNV12Buffer.h", + "objc/api/video_frame_buffer/RTCNativeNV12Buffer.h", + ] + } if (!build_with_chromium) { sources += [ "objc/api/logging/RTCCallbackLogger.h", @@ -1865,7 +1933,6 @@ if (is_ios || is_mac) { rtc_library("native_video") { sources = [ "objc/native/src/objc_frame_buffer.h", - "objc/native/src/objc_frame_buffer.mm", "objc/native/src/objc_video_decoder_factory.h", "objc/native/src/objc_video_decoder_factory.mm", "objc/native/src/objc_video_encoder_factory.h", @@ -1877,6 +1944,15 @@ if (is_ios || is_mac) { "objc/native/src/objc_video_track_source.h", "objc/native/src/objc_video_track_source.mm", ] + if (stream_enable_rendering_backend) { + sources += [ + "objc/native/src/objc_frame_buffer_stream.mm", + "objc/native/src/objc_nv12_conversion.h", + "objc/native/src/objc_nv12_conversion.mm", + ] + } else { + sources += [ "objc/native/src/objc_frame_buffer.mm" ] + } configs += [ "..:common_objc" ] diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h index 62c7554a54..782c0cf259 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h @@ -43,6 +43,13 @@ typedef NS_ENUM(NSInteger, RTC_OBJC_TYPE(RTCRtpMediaType)); @protocol RTC_OBJC_TYPE (RTCAudioProcessingModule); +typedef NS_ENUM(NSInteger, RTC_OBJC_TYPE(RTCFrameBufferPolicy)) { + RTC_OBJC_TYPE(RTCFrameBufferPolicyNone) = 0, + RTC_OBJC_TYPE(RTCFrameBufferPolicyWrapOnlyExistingNV12), + RTC_OBJC_TYPE(RTCFrameBufferPolicyCopyToNV12), + RTC_OBJC_TYPE(RTCFrameBufferPolicyConvertWithPoolToNV12) +}; + RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) : NSObject @@ -71,6 +78,20 @@ RTC_OBJC_EXPORT @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioDeviceModule) *audioDeviceModule; +/** + * Controls how decoded frame buffers are bridged to Objective-C. Default is `none`. + * This can be toggled at runtime. It only takes effect when + * `stream_enable_rendering_backend=true` (RTC_STREAM_RENDERING_BACKEND). + * + * Note: the policy is evaluated per frame. Changing it mid-call can result in + * a mix of I420 and NV12 buffers for in-flight frames. For consistent format, + * set it before starting a call. + * + * Thread-safety: property access is not synchronized; set it from a single + * thread if you need strict consistency. + */ +@property(nonatomic, assign) RTC_OBJC_TYPE(RTCFrameBufferPolicy) frameBufferPolicy; + /** * Valid kind values are kRTCMediaStreamTrackKindAudio and * kRTCMediaStreamTrackKindVideo. diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index 791aba1e11..d81bfd1dc0 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -57,6 +57,9 @@ #include "sdk/objc/native/api/video_encoder_factory.h" #include "sdk/objc/native/src/objc_video_decoder_factory.h" #include "sdk/objc/native/src/objc_video_encoder_factory.h" +#if defined(RTC_STREAM_RENDERING_BACKEND) +#include "sdk/objc/native/src/objc_nv12_conversion.h" +#endif #import "components/audio/RTCAudioProcessingModule.h" #import "components/audio/RTCDefaultAudioProcessingModule+Private.h" @@ -73,6 +76,7 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { RTC_OBJC_TYPE(RTCDefaultAudioProcessingModule) *_defaultAudioProcessingModule; BOOL _hasStartedAecDump; + RTC_OBJC_TYPE(RTCFrameBufferPolicy) _frameBufferPolicy; } @synthesize nativeFactory = _nativeFactory; @@ -196,6 +200,19 @@ - (instancetype)initWithNativeDependencies: return self; } +- (RTC_OBJC_TYPE(RTCFrameBufferPolicy))frameBufferPolicy { + return _frameBufferPolicy; +} + +- (void)setFrameBufferPolicy:(RTC_OBJC_TYPE(RTCFrameBufferPolicy))frameBufferPolicy { + _frameBufferPolicy = frameBufferPolicy; +#if defined(RTC_STREAM_RENDERING_BACKEND) + // Only available in Stream builds with the shared backend enabled. + webrtc::SetObjCFrameBufferPolicy( + static_cast(frameBufferPolicy)); +#endif +} + - (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpSenderCapabilitiesFor:(RTC_OBJC_TYPE(RTCRtpMediaType))mediaType { webrtc::RtpCapabilities capabilities = _nativeFactory->GetRtpSenderCapabilities([RTC_OBJC_TYPE(RTCRtpReceiver) nativeMediaTypeForMediaType: mediaType]); diff --git a/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer+Private.h b/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer+Private.h new file mode 100644 index 0000000000..db24a0a2a0 --- /dev/null +++ b/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNativeNV12Buffer.h" + +#include "api/video/video_frame_buffer.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE(RTCNV12Buffer) () { + @protected + webrtc::scoped_refptr _nv12Buffer; +} + +/** Initialize an RTCNV12Buffer with its backing NV12BufferInterface. */ +- (instancetype)initWithFrameBuffer: + (webrtc::scoped_refptr)nv12Buffer; +- (webrtc::scoped_refptr)nativeNV12Buffer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer.h b/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer.h new file mode 100644 index 0000000000..58fc50b0a6 --- /dev/null +++ b/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer.h @@ -0,0 +1,24 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCNV12Buffer.h" +#import "sdk/objc/base/RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** RTCNV12Buffer implements the RTCNV12Buffer protocol */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE(RTCNV12Buffer) + : NSObject +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer.mm b/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer.mm new file mode 100644 index 0000000000..fcd32d9740 --- /dev/null +++ b/sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer.mm @@ -0,0 +1,93 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNativeNV12Buffer+Private.h" +#import "RTCNativeI420Buffer+Private.h" + +#include "api/video/nv12_buffer.h" + +@implementation RTC_OBJC_TYPE(RTCNV12Buffer) + +- (instancetype)initWithWidth:(int)width height:(int)height { + self = [super init]; + if (self) { + _nv12Buffer = webrtc::NV12Buffer::Create(width, height); + } + return self; +} + +- (instancetype)initWithWidth:(int)width + height:(int)height + strideY:(int)strideY + strideUV:(int)strideUV { + self = [super init]; + if (self) { + _nv12Buffer = + webrtc::NV12Buffer::Create(width, height, strideY, strideUV); + } + return self; +} + +- (instancetype)initWithFrameBuffer: + (webrtc::scoped_refptr)nv12Buffer { + self = [super init]; + if (self) { + _nv12Buffer = nv12Buffer; + } + return self; +} + +- (int)width { + return _nv12Buffer->width(); +} + +- (int)height { + return _nv12Buffer->height(); +} + +- (int)strideY { + return _nv12Buffer->StrideY(); +} + +- (int)strideUV { + return _nv12Buffer->StrideUV(); +} + +- (int)chromaWidth { + return _nv12Buffer->ChromaWidth(); +} + +- (int)chromaHeight { + return _nv12Buffer->ChromaHeight(); +} + +- (const uint8_t *)dataY { + return _nv12Buffer->DataY(); +} + +- (const uint8_t *)dataUV { + return _nv12Buffer->DataUV(); +} + +- (id)toI420 { + webrtc::scoped_refptr buffer = + _nv12Buffer->ToI420(); + RTC_OBJC_TYPE(RTCI420Buffer) *result = + [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:buffer]; + return result; +} + +#pragma mark - Private + +- (webrtc::scoped_refptr)nativeNV12Buffer { + return _nv12Buffer; +} + +@end diff --git a/sdk/objc/base/RTCNV12Buffer.h b/sdk/objc/base/RTCNV12Buffer.h new file mode 100644 index 0000000000..4de72a104a --- /dev/null +++ b/sdk/objc/base/RTCNV12Buffer.h @@ -0,0 +1,31 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCVideoFrameBuffer.h" +#import "sdk/objc/base/RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Protocol for RTCVideoFrameBuffers containing NV12 data. */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE(RTCNV12Buffer) + +@property(nonatomic, readonly) int chromaWidth; +@property(nonatomic, readonly) int chromaHeight; +@property(nonatomic, readonly) const uint8_t *dataY; +@property(nonatomic, readonly) const uint8_t *dataUV; +@property(nonatomic, readonly) int strideY; +@property(nonatomic, readonly) int strideUV; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/renderer/metal/RTCSharedMetalRenderAdapter.h b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderAdapter.h new file mode 100644 index 0000000000..e8258ad674 --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderAdapter.h @@ -0,0 +1,53 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#import "RTCVideoFrame.h" +#import "sdk/objc/base/RTCMacros.h" + +#if TARGET_OS_IPHONE +#import +#endif + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCSharedMetalRenderingContext); +@protocol MTLCommandBuffer; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE(RTCSharedMetalRenderAdapter) : NSObject + +- (instancetype)initWithContext: + (RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context + NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + +- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame; +- (BOOL)consumeNeedsRedraw; +- (nullable RTC_OBJC_TYPE(RTCVideoFrame) *)consumeFrame; + +- (void)setContentMode:(UIViewContentMode)contentMode; +- (void)setMaxInFlightFrames:(NSInteger)maxInFlightFrames; + +- (BOOL)beginFrameIfPossible; +- (void)endFrame; + +- (BOOL)encodeFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame + drawable:(id)drawable + context:(RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context + commandBuffer:(id)commandBuffer + rotationOverride:(nullable NSValue *)rotationOverride; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/renderer/metal/RTCSharedMetalRenderAdapter.mm b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderAdapter.mm new file mode 100644 index 0000000000..165ac4860d --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderAdapter.mm @@ -0,0 +1,544 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCSharedMetalRenderAdapter.h" + +#import +#import + +#import "RTCSharedMetalRenderingContext+Private.h" +#import "base/RTCLogging.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#import "sdk/objc/base/RTCI420Buffer.h" +#import "sdk/objc/base/RTCNV12Buffer.h" + +@implementation RTC_OBJC_TYPE(RTCSharedMetalRenderAdapter) { + dispatch_queue_t _frameQueue; + RTC_OBJC_TYPE(RTCVideoFrame) *_latestFrame; + BOOL _needsRedraw; + UIViewContentMode _contentMode; + NSInteger _inFlightFrames; + NSInteger _maxInFlightFrames; + + id _vertexBuffer; + + id _i420YTexture; + id _i420UTexture; + id _i420VTexture; + CGSize _i420TextureSize; + + id _nv12YTexture; + id _nv12UVTexture; + CGSize _nv12TextureSize; +} + +- (instancetype)initWithContext: + (RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context { + self = [super init]; + if (self) { + _frameQueue = dispatch_queue_create( + "webrtc.sharedmetal.frames", DISPATCH_QUEUE_SERIAL); + _contentMode = UIViewContentModeScaleAspectFill; + _maxInFlightFrames = 0; + + _vertexBuffer = + [context.device newBufferWithLength:16 * sizeof(float) + options:MTLResourceCPUCacheModeWriteCombined]; + if (!_vertexBuffer) { + RTCLogError(@"SharedMetal: Failed to create vertex buffer"); + return nil; + } + } + return self; +} + +- (void)renderFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + dispatch_sync(_frameQueue, ^{ + self->_latestFrame = frame; + self->_needsRedraw = frame != nil; + }); +} + +- (BOOL)consumeNeedsRedraw { + __block BOOL needsRedraw = NO; + dispatch_sync(_frameQueue, ^{ + if (self->_needsRedraw) { + self->_needsRedraw = NO; + needsRedraw = YES; + } + }); + return needsRedraw; +} + +- (RTC_OBJC_TYPE(RTCVideoFrame) *)consumeFrame { + __block RTC_OBJC_TYPE(RTCVideoFrame) *frame = nil; + dispatch_sync(_frameQueue, ^{ + frame = self->_latestFrame; + }); + return frame; +} + +- (void)setContentMode:(UIViewContentMode)contentMode { + dispatch_sync(_frameQueue, ^{ + self->_contentMode = contentMode; + }); +} + +- (void)setMaxInFlightFrames:(NSInteger)maxInFlightFrames { + dispatch_sync(_frameQueue, ^{ + self->_maxInFlightFrames = MAX(0, maxInFlightFrames); + }); +} + +- (BOOL)beginFrameIfPossible { + __block BOOL canBegin = NO; + dispatch_sync(_frameQueue, ^{ + if (self->_maxInFlightFrames > 0 && + self->_inFlightFrames >= self->_maxInFlightFrames) { + canBegin = NO; + return; + } + self->_inFlightFrames += 1; + canBegin = YES; + }); + return canBegin; +} + +- (void)endFrame { + dispatch_sync(_frameQueue, ^{ + self->_inFlightFrames = MAX(0, self->_inFlightFrames - 1); + }); +} + +- (BOOL)encodeFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame + drawable:(id)drawable + context:(RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context + commandBuffer:(id)commandBuffer + rotationOverride:(nullable NSValue *)rotationOverride { + CGSize targetSize = CGSizeMake(drawable.texture.width, + drawable.texture.height); + RTC_OBJC_TYPE(RTCVideoRotation) rotation = frame.rotation; + if (rotationOverride) { + if (@available(iOS 11.0, *)) { + [rotationOverride getValue:&rotation size:sizeof(rotation)]; + } else { + [rotationOverride getValue:&rotation]; + } + } + + MTLRenderPassDescriptor *renderPassDescriptor = + [[MTLRenderPassDescriptor alloc] init]; + renderPassDescriptor.colorAttachments[0].texture = drawable.texture; + renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionClear; + renderPassDescriptor.colorAttachments[0].clearColor = + MTLClearColorMake(0.0, 0.0, 0.0, 1.0); + + id renderEncoder = + [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor]; + if (!renderEncoder) { + return NO; + } + + [renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0]; + + NSMutableArray *textureRefsToRetain = [NSMutableArray array]; + + // Supported formats: CVPixelBuffer (NV12/BGRA), NV12, and I420. + id buffer = frame.buffer; + // Match RTCMTLRenderer's texture mapping to avoid rotation/crop drift. + [self updateVertexBufferForFrame:frame + targetSize:targetSize + rotation:rotation + contentMode:_contentMode]; + if ([buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *pixelBuffer = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)buffer; + OSType pixelFormat = + CVPixelBufferGetPixelFormatType(pixelBuffer.pixelBuffer); + if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || + pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + CVMetalTextureRef yRef = nil; + CVMetalTextureRef uvRef = nil; + if (![self updateNV12TexturesFromPixelBuffer:pixelBuffer.pixelBuffer + context:context + yRef:&yRef + uvRef:&uvRef]) { + [renderEncoder endEncoding]; + return NO; + } + id yTexture = CVMetalTextureGetTexture(yRef); + id uvTexture = CVMetalTextureGetTexture(uvRef); + if (!yTexture || !uvTexture) { + if (yRef) { + CFRelease(yRef); + } + if (uvRef) { + CFRelease(uvRef); + } + [renderEncoder endEncoding]; + return NO; + } + [textureRefsToRetain addObject:(__bridge_transfer id)yRef]; + [textureRefsToRetain addObject:(__bridge_transfer id)uvRef]; + [renderEncoder setRenderPipelineState:context.nv12PipelineState]; + [renderEncoder setFragmentTexture:yTexture atIndex:0]; + [renderEncoder setFragmentTexture:uvTexture atIndex:1]; + } else if (pixelFormat == kCVPixelFormatType_32BGRA) { + CVMetalTextureRef bgraRef = nil; + if (![self updateBGRATextureFromPixelBuffer:pixelBuffer.pixelBuffer + context:context + ref:&bgraRef]) { + [renderEncoder endEncoding]; + return NO; + } + id bgraTexture = CVMetalTextureGetTexture(bgraRef); + if (!bgraTexture) { + if (bgraRef) { + CFRelease(bgraRef); + } + [renderEncoder endEncoding]; + return NO; + } + [textureRefsToRetain addObject:(__bridge_transfer id)bgraRef]; + [renderEncoder setRenderPipelineState:context.bgraPipelineState]; + [renderEncoder setFragmentTexture:bgraTexture atIndex:0]; + } else { + [renderEncoder endEncoding]; + return NO; + } + } else if ([buffer conformsToProtocol:@protocol(RTC_OBJC_TYPE(RTCNV12Buffer))]) { + id nv12Buffer = + (id)buffer; + if (![self updateNV12TexturesFromNV12Buffer:nv12Buffer context:context]) { + [renderEncoder endEncoding]; + return NO; + } + [renderEncoder setRenderPipelineState:context.nv12PipelineState]; + [renderEncoder setFragmentTexture:_nv12YTexture atIndex:0]; + [renderEncoder setFragmentTexture:_nv12UVTexture atIndex:1]; + } else if ([buffer conformsToProtocol:@protocol(RTC_OBJC_TYPE(RTCI420Buffer))]) { + id i420Buffer = + (id)buffer; + if (![self updateI420TexturesFromI420Buffer:i420Buffer context:context]) { + [renderEncoder endEncoding]; + return NO; + } + [renderEncoder setRenderPipelineState:context.i420PipelineState]; + [renderEncoder setFragmentTexture:_i420YTexture atIndex:0]; + [renderEncoder setFragmentTexture:_i420UTexture atIndex:1]; + [renderEncoder setFragmentTexture:_i420VTexture atIndex:2]; + } else { + id i420Buffer = [buffer toI420]; + if (!i420Buffer || + ![self updateI420TexturesFromI420Buffer:i420Buffer context:context]) { + [renderEncoder endEncoding]; + return NO; + } + [renderEncoder setRenderPipelineState:context.i420PipelineState]; + [renderEncoder setFragmentTexture:_i420YTexture atIndex:0]; + [renderEncoder setFragmentTexture:_i420UTexture atIndex:1]; + [renderEncoder setFragmentTexture:_i420VTexture atIndex:2]; + } + + [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip + vertexStart:0 + vertexCount:4 + instanceCount:1]; + [renderEncoder endEncoding]; + + if (textureRefsToRetain.count > 0) { + [commandBuffer addCompletedHandler:^(__unused id buffer) { + (void)textureRefsToRetain; + }]; + } + + return YES; +} + +#pragma mark - Private + +- (void)updateVertexBufferForFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame + targetSize:(CGSize)targetSize + rotation:(RTC_OBJC_TYPE(RTCVideoRotation))rotation + contentMode:(UIViewContentMode)contentMode { + // Default to the full buffer size for non-CVPixelBuffer inputs. + int frameWidth = frame.buffer.width; + int frameHeight = frame.buffer.height; + int cropX = 0; + int cropY = 0; + int cropWidth = frameWidth; + int cropHeight = frameHeight; + + // CVPixelBuffer carries a crop rect that must be respected, otherwise + // the image will appear stretched or off-center. + if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *pixelBuffer = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + frameWidth = (int)CVPixelBufferGetWidth(pixelBuffer.pixelBuffer); + frameHeight = (int)CVPixelBufferGetHeight(pixelBuffer.pixelBuffer); + cropX = pixelBuffer.cropX; + cropY = pixelBuffer.cropY; + cropWidth = pixelBuffer.cropWidth; + cropHeight = pixelBuffer.cropHeight; + } + + // Guard against invalid buffers to avoid corrupting the vertex buffer. + if (frameWidth <= 0 || frameHeight <= 0) { + return; + } + + // Treat empty crops as full-frame to keep texture coords in range. + if (cropWidth <= 0 || cropHeight <= 0) { + cropX = 0; + cropY = 0; + cropWidth = frameWidth; + cropHeight = frameHeight; + } + + CGFloat targetAspect = targetSize.width / MAX(targetSize.height, 1); + // When rotated 90/270, the source aspect is effectively swapped. + CGFloat sourceAspect = + (rotation == RTC_OBJC_TYPE(RTCVideoRotation_90) || + rotation == RTC_OBJC_TYPE(RTCVideoRotation_270)) + ? (CGFloat)cropHeight / MAX(cropWidth, 1) + : (CGFloat)cropWidth / MAX(cropHeight, 1); + + // Normalize crop rect to texture coordinates (0..1). + CGFloat uMin = (CGFloat)cropX / frameWidth; + CGFloat uMax = (CGFloat)(cropX + cropWidth) / frameWidth; + CGFloat vMin = (CGFloat)cropY / frameHeight; + CGFloat vMax = (CGFloat)(cropY + cropHeight) / frameHeight; + CGFloat xScale = 1; + CGFloat yScale = 1; + + if (contentMode == UIViewContentModeScaleAspectFit) { + // Letterbox/pillarbox via view-space scaling. + if (targetAspect > sourceAspect) { + xScale = sourceAspect / targetAspect; + } else { + yScale = targetAspect / sourceAspect; + } + } else { + // Center-crop by trimming texture coordinates. + if (targetAspect > sourceAspect) { + CGFloat vSpan = vMax - vMin; + CGFloat vCrop = vSpan * (sourceAspect / targetAspect); + CGFloat vOffset = (vSpan - vCrop) / 2; + vMin = vMin + vOffset; + vMax = vMin + vCrop; + } else { + CGFloat uSpan = uMax - uMin; + CGFloat uCrop = uSpan * (targetAspect / sourceAspect); + CGFloat uOffset = (uSpan - uCrop) / 2; + uMin = uMin + uOffset; + uMax = uMin + uCrop; + } + } + + // Use the same UV ordering as RTCMTLRenderer for each rotation. + float vertices[16]; + switch (rotation) { + case RTC_OBJC_TYPE(RTCVideoRotation_90): { + float values[16] = { + -static_cast(xScale), -static_cast(yScale), + static_cast(uMax), static_cast(vMax), + static_cast(xScale), -static_cast(yScale), + static_cast(uMax), static_cast(vMin), + -static_cast(xScale), static_cast(yScale), + static_cast(uMin), static_cast(vMax), + static_cast(xScale), static_cast(yScale), + static_cast(uMin), static_cast(vMin) + }; + memcpy(vertices, values, sizeof(values)); + break; + } + case RTC_OBJC_TYPE(RTCVideoRotation_180): { + float values[16] = { + -static_cast(xScale), -static_cast(yScale), + static_cast(uMax), static_cast(vMin), + static_cast(xScale), -static_cast(yScale), + static_cast(uMin), static_cast(vMin), + -static_cast(xScale), static_cast(yScale), + static_cast(uMax), static_cast(vMax), + static_cast(xScale), static_cast(yScale), + static_cast(uMin), static_cast(vMax) + }; + memcpy(vertices, values, sizeof(values)); + break; + } + case RTC_OBJC_TYPE(RTCVideoRotation_270): { + float values[16] = { + -static_cast(xScale), -static_cast(yScale), + static_cast(uMin), static_cast(vMin), + static_cast(xScale), -static_cast(yScale), + static_cast(uMin), static_cast(vMax), + -static_cast(xScale), static_cast(yScale), + static_cast(uMax), static_cast(vMin), + static_cast(xScale), static_cast(yScale), + static_cast(uMax), static_cast(vMax) + }; + memcpy(vertices, values, sizeof(values)); + break; + } + default: { + float values[16] = { + -static_cast(xScale), -static_cast(yScale), + static_cast(uMin), static_cast(vMax), + static_cast(xScale), -static_cast(yScale), + static_cast(uMax), static_cast(vMax), + -static_cast(xScale), static_cast(yScale), + static_cast(uMin), static_cast(vMin), + static_cast(xScale), static_cast(yScale), + static_cast(uMax), static_cast(vMin) + }; + memcpy(vertices, values, sizeof(values)); + break; + } + } + + memcpy(_vertexBuffer.contents, vertices, sizeof(vertices)); +} + +- (BOOL)updateNV12TexturesFromPixelBuffer:(CVPixelBufferRef)pixelBuffer + context: + (RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context + yRef:(CVMetalTextureRef *)yRef + uvRef:(CVMetalTextureRef *)uvRef { + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + + CVReturn yResult = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, context.textureCache, pixelBuffer, nil, + MTLPixelFormatR8Unorm, width, height, 0, yRef); + CVReturn uvResult = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, context.textureCache, pixelBuffer, nil, + MTLPixelFormatRG8Unorm, width / 2, height / 2, 1, uvRef); + + return yResult == kCVReturnSuccess && + uvResult == kCVReturnSuccess && *yRef != nil && *uvRef != nil; +} + +- (BOOL)updateBGRATextureFromPixelBuffer:(CVPixelBufferRef)pixelBuffer + context: + (RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context + ref:(CVMetalTextureRef *)ref { + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + + CVReturn result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, context.textureCache, pixelBuffer, nil, + MTLPixelFormatBGRA8Unorm, width, height, 0, ref); + + return result == kCVReturnSuccess && *ref != nil; +} + +- (BOOL)updateI420TexturesFromI420Buffer: + (id)buffer + context: + (RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context { + CGSize size = CGSizeMake(buffer.width, buffer.height); + if (!CGSizeEqualToSize(size, _i420TextureSize) || + _i420YTexture == nil) { + MTLTextureDescriptor *yDescriptor = + [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm + width:buffer.width + height:buffer.height + mipmapped:NO]; + yDescriptor.usage = MTLTextureUsageShaderRead; + + MTLTextureDescriptor *uvDescriptor = + [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm + width:buffer.width / 2 + height:buffer.height / 2 + mipmapped:NO]; + uvDescriptor.usage = MTLTextureUsageShaderRead; + + _i420YTexture = [context.device newTextureWithDescriptor:yDescriptor]; + _i420UTexture = [context.device newTextureWithDescriptor:uvDescriptor]; + _i420VTexture = [context.device newTextureWithDescriptor:uvDescriptor]; + _i420TextureSize = size; + } + + if (!_i420YTexture || !_i420UTexture || !_i420VTexture) { + return NO; + } + + MTLRegion yRegion = + MTLRegionMake2D(0, 0, _i420YTexture.width, _i420YTexture.height); + [_i420YTexture replaceRegion:yRegion + mipmapLevel:0 + withBytes:buffer.dataY + bytesPerRow:buffer.strideY]; + + MTLRegion uRegion = + MTLRegionMake2D(0, 0, _i420UTexture.width, _i420UTexture.height); + [_i420UTexture replaceRegion:uRegion + mipmapLevel:0 + withBytes:buffer.dataU + bytesPerRow:buffer.strideU]; + + MTLRegion vRegion = + MTLRegionMake2D(0, 0, _i420VTexture.width, _i420VTexture.height); + [_i420VTexture replaceRegion:vRegion + mipmapLevel:0 + withBytes:buffer.dataV + bytesPerRow:buffer.strideV]; + + return YES; +} + +- (BOOL)updateNV12TexturesFromNV12Buffer: + (id)buffer + context: + (RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *)context { + CGSize size = CGSizeMake(buffer.width, buffer.height); + if (!CGSizeEqualToSize(size, _nv12TextureSize) || + _nv12YTexture == nil) { + MTLTextureDescriptor *yDescriptor = + [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm + width:buffer.width + height:buffer.height + mipmapped:NO]; + yDescriptor.usage = MTLTextureUsageShaderRead; + + MTLTextureDescriptor *uvDescriptor = + [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRG8Unorm + width:buffer.width / 2 + height:buffer.height / 2 + mipmapped:NO]; + uvDescriptor.usage = MTLTextureUsageShaderRead; + + _nv12YTexture = [context.device newTextureWithDescriptor:yDescriptor]; + _nv12UVTexture = [context.device newTextureWithDescriptor:uvDescriptor]; + _nv12TextureSize = size; + } + + if (!_nv12YTexture || !_nv12UVTexture) { + return NO; + } + + MTLRegion yRegion = + MTLRegionMake2D(0, 0, _nv12YTexture.width, _nv12YTexture.height); + [_nv12YTexture replaceRegion:yRegion + mipmapLevel:0 + withBytes:buffer.dataY + bytesPerRow:buffer.strideY]; + + MTLRegion uvRegion = + MTLRegionMake2D(0, 0, _nv12UVTexture.width, _nv12UVTexture.height); + [_nv12UVTexture replaceRegion:uvRegion + mipmapLevel:0 + withBytes:buffer.dataUV + bytesPerRow:buffer.strideUV]; + + return YES; +} + +@end diff --git a/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext+Private.h b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext+Private.h new file mode 100644 index 0000000000..5d8b89c94f --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCSharedMetalRenderingContext.h" + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) () + +@property(nonatomic, readonly) id device; +@property(nonatomic, readonly) id commandQueue; +@property(nonatomic, readonly) CVMetalTextureCacheRef textureCache; +@property(nonatomic, readonly) id nv12PipelineState; +@property(nonatomic, readonly) id i420PipelineState; +@property(nonatomic, readonly) id bgraPipelineState; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext.h b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext.h new file mode 100644 index 0000000000..cf227dd8c0 --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext.h @@ -0,0 +1,34 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "sdk/objc/base/RTCMacros.h" + +#if TARGET_OS_IPHONE +#import +#endif + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCSharedMetalVideoView); + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) : NSObject + +// Shared Metal context used by RTCSharedMetalVideoView. iOS-only. ++ (nullable instancetype)sharedContext; + +- (void)registerView:(RTC_OBJC_TYPE(RTCSharedMetalVideoView) *)view; +- (void)unregisterView:(RTC_OBJC_TYPE(RTCSharedMetalVideoView) *)view; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext.mm b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext.mm new file mode 100644 index 0000000000..81884b83d8 --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCSharedMetalRenderingContext.mm @@ -0,0 +1,261 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCSharedMetalRenderingContext.h" + +#import +#import +#import + +#import "RTCSharedMetalRenderingContext+Private.h" +#import "RTCSharedMetalVideoView.h" +#import "base/RTCLogging.h" + +// Minimal Metal shader source for YUV/BGRA sampling used by the shared backend. +static NSString *const kSharedMetalShaderSource = + @"#include \n" + "using namespace metal;\n" + "\n" + "struct VertexOut {\n" + " float4 position [[position]];\n" + " float2 texcoord;\n" + "};\n" + "\n" + "vertex VertexOut streamVideoVertex(\n" + " uint vertexID [[vertex_id]],\n" + " const device float4 *vertices [[buffer(0)]]) {\n" + " float4 data = vertices[vertexID];\n" + " VertexOut out;\n" + " out.position = float4(data.xy, 0.0, 1.0);\n" + " out.texcoord = data.zw;\n" + " return out;\n" + "}\n" + "\n" + "fragment float4 streamVideoNV12Fragment(\n" + " VertexOut in [[stage_in]],\n" + " texture2d yTexture [[texture(0)]],\n" + " texture2d uvTexture [[texture(1)]]) {\n" + " constexpr sampler samplerState(address::clamp_to_edge, filter::linear);\n" + " float y = yTexture.sample(samplerState, in.texcoord).r;\n" + " float2 uv = uvTexture.sample(samplerState, in.texcoord).rg - float2(0.5, 0.5);\n" + "\n" + " float3 rgb;\n" + " rgb.r = y + 1.402 * uv.y;\n" + " rgb.g = y - 0.344136 * uv.x - 0.714136 * uv.y;\n" + " rgb.b = y + 1.772 * uv.x;\n" + "\n" + " return float4(rgb, 1.0);\n" + "}\n" + "\n" + "fragment float4 streamVideoI420Fragment(\n" + " VertexOut in [[stage_in]],\n" + " texture2d yTexture [[texture(0)]],\n" + " texture2d uTexture [[texture(1)]],\n" + " texture2d vTexture [[texture(2)]]) {\n" + " constexpr sampler samplerState(address::clamp_to_edge, filter::linear);\n" + " float y = yTexture.sample(samplerState, in.texcoord).r;\n" + " float u = uTexture.sample(samplerState, in.texcoord).r - 0.5;\n" + " float v = vTexture.sample(samplerState, in.texcoord).r - 0.5;\n" + "\n" + " float3 rgb;\n" + " rgb.r = y + 1.402 * v;\n" + " rgb.g = y - 0.344136 * u - 0.714136 * v;\n" + " rgb.b = y + 1.772 * u;\n" + "\n" + " return float4(rgb, 1.0);\n" + "}\n" + "\n" + "fragment float4 streamVideoBGRAFragment(\n" + " VertexOut in [[stage_in]],\n" + " texture2d texture [[texture(0)]]) {\n" + " constexpr sampler samplerState(address::clamp_to_edge, filter::linear);\n" + " return texture.sample(samplerState, in.texcoord);\n" + "}\n"; + +@implementation RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) { + // Serial queue to guard the weak view set. + dispatch_queue_t _viewsQueue; + NSHashTable *_views; + CADisplayLink *_displayLink; +} + ++ (nullable instancetype)sharedContext { + static RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *sharedContext = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + sharedContext = [[RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) alloc] init]; + if (!sharedContext) { + RTCLogError(@"SharedMetal: Failed to initialize shared context; " + @"falling back to per-view rendering for this process."); + } + }); + return sharedContext; +} + +- (instancetype)init { + self = [super init]; + if (self) { + _viewsQueue = dispatch_queue_create( + "webrtc.sharedmetal.views", DISPATCH_QUEUE_SERIAL); + _views = [NSHashTable weakObjectsHashTable]; + + _device = MTLCreateSystemDefaultDevice(); + if (!_device) { + RTCLogError(@"SharedMetal: Failed to create MTLDevice"); + return nil; + } + + _commandQueue = [_device newCommandQueue]; + if (!_commandQueue) { + RTCLogError(@"SharedMetal: Failed to create command queue"); + return nil; + } + + CVMetalTextureCacheRef cache = nil; + CVReturn cacheResult = CVMetalTextureCacheCreate( + kCFAllocatorDefault, nil, _device, nil, &cache); + if (cacheResult != kCVReturnSuccess || cache == nil) { + RTCLogError(@"SharedMetal: Failed to create texture cache"); + return nil; + } + _textureCache = cache; + + id library = [self makeLibraryWithDevice:_device]; + if (!library) { + return nil; + } + + _nv12PipelineState = + [self makePipelineStateWithDevice:_device + library:library + fragmentName:@"streamVideoNV12Fragment"]; + _i420PipelineState = + [self makePipelineStateWithDevice:_device + library:library + fragmentName:@"streamVideoI420Fragment"]; + _bgraPipelineState = + [self makePipelineStateWithDevice:_device + library:library + fragmentName:@"streamVideoBGRAFragment"]; + + if (!_nv12PipelineState || !_i420PipelineState || !_bgraPipelineState) { + RTCLogError(@"SharedMetal: Failed to create pipeline states"); + return nil; + } + } + + return self; +} + +- (void)dealloc { + if (_textureCache) { + CFRelease(_textureCache); + _textureCache = nil; + } + [self stopDisplayLinkIfNeeded]; +} + +- (void)registerView:(RTC_OBJC_TYPE(RTCSharedMetalVideoView) *)view { + dispatch_sync(_viewsQueue, ^{ + [self->_views addObject:view]; + }); + [self startDisplayLinkIfNeeded]; +} + +- (void)unregisterView:(RTC_OBJC_TYPE(RTCSharedMetalVideoView) *)view { + dispatch_sync(_viewsQueue, ^{ + [self->_views removeObject:view]; + }); + [self stopDisplayLinkIfNeeded]; +} + +#pragma mark - Private + +- (void)startDisplayLinkIfNeeded { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_displayLink) { + return; + } + self->_displayLink = [CADisplayLink displayLinkWithTarget:self + selector:@selector(displayLinkDidFire)]; + [self->_displayLink addToRunLoop:[NSRunLoop mainRunLoop] + forMode:NSRunLoopCommonModes]; + }); +} + +- (void)stopDisplayLinkIfNeeded { + dispatch_async(dispatch_get_main_queue(), ^{ + __block BOOL hasViews = NO; + dispatch_sync(self->_viewsQueue, ^{ + hasViews = self->_views.count > 0; + }); + if (!hasViews) { + [self->_displayLink invalidate]; + self->_displayLink = nil; + } + }); +} + +- (void)displayLinkDidFire { + __block NSArray *viewsSnapshot = nil; + dispatch_sync(_viewsQueue, ^{ + viewsSnapshot = self->_views.allObjects; + }); + + for (RTC_OBJC_TYPE(RTCSharedMetalVideoView) *view in viewsSnapshot) { + [view drawIfNeeded]; + } +} + +- (nullable id)makeLibraryWithDevice:(id)device { + if (@available(iOS 10.0, *)) { + NSError *error = nil; + id library = + [device newLibraryWithSource:kSharedMetalShaderSource + options:nil + error:&error]; + if (!library) { + RTCLogError(@"SharedMetal: Failed to compile shaders: %@", error); + } + return library; + } + RTCLogError(@"SharedMetal: Metal shader compilation requires iOS 10+"); + return nil; +} + +- (nullable id) + makePipelineStateWithDevice:(id)device + library:(id)library + fragmentName:(NSString *)fragmentName { + id vertexFunction = + [library newFunctionWithName:@"streamVideoVertex"]; + id fragmentFunction = + [library newFunctionWithName:fragmentName]; + if (!vertexFunction || !fragmentFunction) { + RTCLogError(@"SharedMetal: Missing Metal functions for %@", fragmentName); + return nil; + } + + MTLRenderPipelineDescriptor *descriptor = + [[MTLRenderPipelineDescriptor alloc] init]; + descriptor.vertexFunction = vertexFunction; + descriptor.fragmentFunction = fragmentFunction; + descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + + NSError *error = nil; + id pipelineState = + [device newRenderPipelineStateWithDescriptor:descriptor error:&error]; + if (!pipelineState) { + RTCLogError(@"SharedMetal: Failed to create pipeline state: %@", error); + } + return pipelineState; +} + +@end diff --git a/sdk/objc/components/renderer/metal/RTCSharedMetalVideoView.h b/sdk/objc/components/renderer/metal/RTCSharedMetalVideoView.h new file mode 100644 index 0000000000..27870d0d44 --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCSharedMetalVideoView.h @@ -0,0 +1,33 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCVideoRenderer.h" +#import "sdk/objc/base/RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE(RTCSharedMetalVideoView) + : UIView + +@property(nonatomic, weak) id delegate; +@property(nonatomic) UIViewContentMode videoContentMode; +@property(nonatomic, getter=isEnabled) BOOL enabled; +@property(nonatomic, nullable) NSValue *rotationOverride; +// Limits in-flight frames for the shared Metal pipeline. +@property(nonatomic, assign) NSInteger maxInFlightFrames; + +- (void)drawIfNeeded; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/renderer/metal/RTCSharedMetalVideoView.mm b/sdk/objc/components/renderer/metal/RTCSharedMetalVideoView.mm new file mode 100644 index 0000000000..efcf69e556 --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCSharedMetalVideoView.mm @@ -0,0 +1,186 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCSharedMetalVideoView.h" + +#import +#import +#import + +#import "RTCSharedMetalRenderAdapter.h" +#import "RTCSharedMetalRenderingContext.h" +#import "RTCSharedMetalRenderingContext+Private.h" +#import "base/RTCLogging.h" + +@implementation RTC_OBJC_TYPE(RTCSharedMetalVideoView) { + RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) *_context; + RTC_OBJC_TYPE(RTCSharedMetalRenderAdapter) *_adapter; + os_unfair_lock _stateLock; + BOOL _renderActive; +} + ++ (Class)layerClass { + return [CAMetalLayer class]; +} + +- (instancetype)initWithFrame:(CGRect)frame { + self = [super initWithFrame:frame]; + if (self) { + _context = [RTC_OBJC_TYPE(RTCSharedMetalRenderingContext) sharedContext]; + if (!_context) { + RTCLogError(@"SharedMetal: Failed to create rendering context"); + return nil; + } + _adapter = [[RTC_OBJC_TYPE(RTCSharedMetalRenderAdapter) alloc] + initWithContext:_context]; + if (!_adapter) { + RTCLogError(@"SharedMetal: Failed to create render adapter"); + return nil; + } + + _enabled = YES; + _renderActive = YES; + _videoContentMode = UIViewContentModeScaleAspectFill; + _maxInFlightFrames = 0; + _stateLock = OS_UNFAIR_LOCK_INIT; + + [self configureLayer]; + [_adapter setContentMode:_videoContentMode]; + [_adapter setMaxInFlightFrames:_maxInFlightFrames]; + // Registers the view with the shared display-link renderer. + [_context registerView:self]; + } + return self; +} + +- (void)dealloc { + [_context unregisterView:self]; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + [self updateDrawableSizeWithSize:self.bounds.size]; +} + +- (void)setEnabled:(BOOL)enabled { + _enabled = enabled; + os_unfair_lock_lock(&_stateLock); + _renderActive = enabled; + os_unfair_lock_unlock(&_stateLock); +} + +- (void)setVideoContentMode:(UIViewContentMode)videoContentMode { + _videoContentMode = videoContentMode; + [_adapter setContentMode:videoContentMode]; +} + +- (void)setMaxInFlightFrames:(NSInteger)maxInFlightFrames { + _maxInFlightFrames = MAX(0, maxInFlightFrames); + [_adapter setMaxInFlightFrames:_maxInFlightFrames]; +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +- (void)setSize:(CGSize)size { + __weak RTC_OBJC_TYPE(RTCSharedMetalVideoView) *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + RTC_OBJC_TYPE(RTCSharedMetalVideoView) *strongSelf = weakSelf; + if (!strongSelf) { + return; + } + // Keep drawable size tied to view bounds; setSize conveys frame size only. + [strongSelf updateDrawableSizeWithSize:strongSelf.bounds.size]; + [strongSelf.delegate videoView:strongSelf didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + if (!self.isEnabled) { + return; + } + [_adapter renderFrame:frame]; +} + +#pragma mark - Rendering + +- (void)drawIfNeeded { + os_unfair_lock_lock(&_stateLock); + BOOL isActive = _renderActive; + os_unfair_lock_unlock(&_stateLock); + if (!isActive) { + return; + } + if (![_adapter consumeNeedsRedraw]) { + return; + } + if (![_adapter beginFrameIfPossible]) { + return; + } + id drawable = [self metalLayer].nextDrawable; + if (!drawable) { + [_adapter endFrame]; + return; + } + RTC_OBJC_TYPE(RTCVideoFrame) *frame = [_adapter consumeFrame]; + if (!frame) { + [_adapter endFrame]; + return; + } + id commandBuffer = + [_context.commandQueue commandBuffer]; + if (!commandBuffer) { + [_adapter endFrame]; + return; + } + + BOOL encoded = [_adapter encodeFrame:frame + drawable:drawable + context:_context + commandBuffer:commandBuffer + rotationOverride:self.rotationOverride]; + if (!encoded) { + [_adapter endFrame]; + return; + } + + [commandBuffer presentDrawable:drawable]; + + RTC_OBJC_TYPE(RTCSharedMetalRenderAdapter) *adapter = _adapter; + [commandBuffer addCompletedHandler:^(__unused id buffer) { + [adapter endFrame]; + }]; + + [commandBuffer commit]; +} + +#pragma mark - Private + +- (CAMetalLayer *)metalLayer { + return (CAMetalLayer *)self.layer; +} + +- (void)configureLayer { + CAMetalLayer *layer = [self metalLayer]; + layer.device = _context.device; + layer.pixelFormat = MTLPixelFormatBGRA8Unorm; + layer.framebufferOnly = YES; + layer.opaque = YES; + layer.contentsScale = UIScreen.mainScreen.scale; + [self updateDrawableSizeWithSize:self.bounds.size]; +} + +- (void)updateDrawableSizeWithSize:(CGSize)size { + CAMetalLayer *layer = [self metalLayer]; + CGFloat scale = self.window ? self.window.screen.scale : UIScreen.mainScreen.scale; + layer.frame = self.bounds; + layer.drawableSize = CGSizeMake(size.width * scale, size.height * scale); +} + +@end diff --git a/sdk/objc/components/renderer/metal/RTCVideoRenderingView.h b/sdk/objc/components/renderer/metal/RTCVideoRenderingView.h new file mode 100644 index 0000000000..3a6d9e6ff7 --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCVideoRenderingView.h @@ -0,0 +1,66 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#if TARGET_OS_OSX +#import +#endif + +#import "RTCVideoFrame.h" +#import "RTCVideoRenderer.h" +#import "sdk/objc/base/RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef NS_ENUM(NSInteger, RTC_OBJC_TYPE(RTCVideoRenderingBackend)) { + RTC_OBJC_TYPE(RTCVideoRenderingBackendDefault) = 0, + // Shared Metal backend is currently iOS-only and is gated by + // RTC_STREAM_RENDERING_BACKEND. On other platforms it falls back to default. + RTC_OBJC_TYPE(RTCVideoRenderingBackendSharedMetal) +}; + +#if TARGET_OS_IPHONE +NS_CLASS_AVAILABLE_IOS(9) +#elif TARGET_OS_OSX +NS_AVAILABLE_MAC(10.11) +#endif + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE(RTCVideoRenderingView) : +#if TARGET_OS_IPHONE + UIView +#elif TARGET_OS_OSX + NSView +#endif + +@property(nonatomic, weak) id delegate; + +#if TARGET_OS_IPHONE +@property(nonatomic) UIViewContentMode videoContentMode; +#endif + +@property(nonatomic, getter=isEnabled) BOOL enabled; +// Wraps an RTCVideoRotation value. Set to nil to use the frame's rotation. +// Swift example: +// let rotation = RTCVideoRotation._90 +// view.rotationOverride = NSNumber(value: rotation.rawValue) +@property(nonatomic, nullable) NSValue *rotationOverride; +// Limits the number of GPU command buffers allowed in flight at once. +// Default is 0 (unlimited). Higher values allow more buffering at the cost +// of extra latency/memory; lower values reduce latency and can drop frames +// when the renderer is saturated. Ignored by backends that do not support it. +@property(nonatomic, assign) NSInteger maxInFlightFrames; + +@property(nonatomic, assign) RTC_OBJC_TYPE(RTCVideoRenderingBackend) renderingBackend; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/renderer/metal/RTCVideoRenderingView.m b/sdk/objc/components/renderer/metal/RTCVideoRenderingView.m new file mode 100644 index 0000000000..384f5c2ad2 --- /dev/null +++ b/sdk/objc/components/renderer/metal/RTCVideoRenderingView.m @@ -0,0 +1,183 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoRenderingView.h" + +#import "RTCMTLVideoView.h" +#if TARGET_OS_IPHONE && defined(RTC_STREAM_RENDERING_BACKEND) +#import "RTCSharedMetalVideoView.h" +#endif + +#if TARGET_OS_IPHONE +#import +#endif + +#if TARGET_OS_IPHONE +typedef UIView RTCVideoRenderingBackendView; +#elif TARGET_OS_OSX +typedef NSView RTCVideoRenderingBackendView; +#endif + +@protocol RTCVideoRenderingBackendConfigurable +@property(nonatomic, weak) id delegate; +@property(nonatomic, getter=isEnabled) BOOL enabled; +@property(nonatomic, nullable) NSValue *rotationOverride; +@property(nonatomic, assign) NSInteger maxInFlightFrames; +#if TARGET_OS_IPHONE +@property(nonatomic) UIViewContentMode videoContentMode; +#endif +@end + +@interface RTC_OBJC_TYPE(RTCVideoRenderingView) () + +@property(nonatomic, strong) RTCVideoRenderingBackendView *activeView; + +@end + +@implementation RTC_OBJC_TYPE(RTCVideoRenderingView) + +- (instancetype)initWithFrame:(CGRect)frame { + self = [super initWithFrame:frame]; + if (self) { + _renderingBackend = RTC_OBJC_TYPE(RTCVideoRenderingBackendDefault); + _enabled = YES; + _maxInFlightFrames = 0; +#if TARGET_OS_IPHONE + _videoContentMode = UIViewContentModeScaleAspectFill; +#endif + [self rebuildActiveView]; + } + return self; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + self.activeView.frame = self.bounds; +} + +- (void)setRenderingBackend:(RTC_OBJC_TYPE(RTCVideoRenderingBackend))renderingBackend { + if (_renderingBackend == renderingBackend) { + return; + } + _renderingBackend = renderingBackend; + [self rebuildActiveView]; +} + +- (void)setEnabled:(BOOL)enabled { + _enabled = enabled; + if ([self.activeView respondsToSelector:@selector(setEnabled:)]) { + id configurable = (id)self.activeView; + configurable.enabled = enabled; + } +} + +#if TARGET_OS_IPHONE +- (void)setVideoContentMode:(UIViewContentMode)videoContentMode { + _videoContentMode = videoContentMode; + if ([self.activeView respondsToSelector:@selector(setVideoContentMode:)]) { + id configurable = (id)self.activeView; + configurable.videoContentMode = videoContentMode; + } +} +#endif + +- (void)setRotationOverride:(NSValue *)rotationOverride { + _rotationOverride = rotationOverride; + if ([self.activeView respondsToSelector:@selector(setRotationOverride:)]) { + id configurable = (id)self.activeView; + configurable.rotationOverride = rotationOverride; + } +} + +- (void)setMaxInFlightFrames:(NSInteger)maxInFlightFrames { + // Only shared-metal uses this; other backends ignore it. + _maxInFlightFrames = MAX(0, maxInFlightFrames); + if ([self.activeView respondsToSelector:@selector(setMaxInFlightFrames:)]) { + id configurable = (id)self.activeView; + configurable.maxInFlightFrames = _maxInFlightFrames; + } +} + +- (void)setDelegate:(id)delegate { + _delegate = delegate; + if ([self.activeView respondsToSelector:@selector(setDelegate:)]) { + id configurable = (id)self.activeView; + configurable.delegate = delegate; + } +} + +#pragma mark - RTCVideoRenderer + +- (void)setSize:(CGSize)size { + [self.activeView setSize:size]; +} + +- (void)renderFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + [self.activeView renderFrame:frame]; +} + +#pragma mark - Private + +- (void)rebuildActiveView { + [self.activeView removeFromSuperview]; + self.activeView = [self createViewForBackend:_renderingBackend]; + self.activeView.frame = self.bounds; + [self addSubview:self.activeView]; + + if ([self.activeView respondsToSelector:@selector(setEnabled:)]) { + id configurable = (id)self.activeView; + configurable.enabled = _enabled; + } + if ([self.activeView respondsToSelector:@selector(setRotationOverride:)]) { + id configurable = (id)self.activeView; + configurable.rotationOverride = _rotationOverride; + } + if ([self.activeView respondsToSelector:@selector(setDelegate:)]) { + id configurable = (id)self.activeView; + configurable.delegate = _delegate; + } + if ([self.activeView respondsToSelector:@selector(setMaxInFlightFrames:)]) { + id configurable = (id)self.activeView; + configurable.maxInFlightFrames = _maxInFlightFrames; + } +#if TARGET_OS_IPHONE + if ([self.activeView respondsToSelector:@selector(setVideoContentMode:)]) { + id configurable = (id)self.activeView; + configurable.videoContentMode = _videoContentMode; + } +#endif +} + +- (RTCVideoRenderingBackendView *)createViewForBackend: + (RTC_OBJC_TYPE(RTCVideoRenderingBackend))backend { + // Shared metal is iOS-only and gated by RTC_STREAM_RENDERING_BACKEND. If it's + // not available we fall back to RTCMTLVideoView. + switch (backend) { + case RTC_OBJC_TYPE(RTCVideoRenderingBackendSharedMetal): +#if TARGET_OS_IPHONE +#if defined(RTC_STREAM_RENDERING_BACKEND) + { + RTC_OBJC_TYPE(RTCSharedMetalVideoView) *view = + [[RTC_OBJC_TYPE(RTCSharedMetalVideoView) alloc] + initWithFrame:self.bounds]; + if (view) { + return view; + } + } +#endif +#endif + return [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:self.bounds]; + case RTC_OBJC_TYPE(RTCVideoRenderingBackendDefault): + default: + return [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:self.bounds]; + } +} + +@end diff --git a/sdk/objc/native/src/objc_frame_buffer_stream.mm b/sdk/objc/native/src/objc_frame_buffer_stream.mm new file mode 100644 index 0000000000..06205e75a8 --- /dev/null +++ b/sdk/objc/native/src/objc_frame_buffer_stream.mm @@ -0,0 +1,314 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Stream-specific alternate implementation that supports NV12 policies. +// This file is only compiled when stream_enable_rendering_backend is enabled. +// It keeps the default objc_frame_buffer.mm untouched and swaps at build time. + +#include "sdk/objc/native/src/objc_frame_buffer.h" + +// Ref-count helper for wrapping ObjC buffers into WebRTC interfaces. +#include "api/make_ref_counted.h" +// NV12 buffer type used when we choose to wrap or convert. +#include "api/video/nv12_buffer.h" +// Pool used by the pooled conversion policy to avoid per-frame allocs. +#include "common_video/include/video_frame_buffer_pool.h" +// Conversion helper for I420 -> NV12 when using the pooled path. +#include "third_party/libyuv/include/libyuv/convert.h" +#include +#import +#import "base/RTCVideoFrameBuffer.h" +#import "sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer+Private.h" +#import "sdk/objc/api/video_frame_buffer/RTCNativeNV12Buffer+Private.h" +#import "sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h" +#include "sdk/objc/native/src/objc_nv12_conversion.h" + +namespace webrtc { + +namespace { +// Avoid NV12 conversion for tiny frames where the overhead is not worth it. +constexpr int kMinNV12ConversionPixels = 64 * 64; + +/** ObjCFrameBuffer that conforms to I420BufferInterface by wrapping + * RTC_OBJC_TYPE(RTCI420Buffer) */ +class ObjCI420FrameBuffer : public I420BufferInterface { + public: + // Stores the ObjC buffer and caches dimensions for quick access. + explicit ObjCI420FrameBuffer(id frame_buffer) + : frame_buffer_(frame_buffer), + width_(frame_buffer.width), + height_(frame_buffer.height) {} + ~ObjCI420FrameBuffer() override {} + + // WebRTC expects dimensions without rotation. + int width() const override { return width_; } + + int height() const override { return height_; } + + // Forward plane pointers and strides to the ObjC buffer. + const uint8_t* DataY() const override { return frame_buffer_.dataY; } + + const uint8_t* DataU() const override { return frame_buffer_.dataU; } + + const uint8_t* DataV() const override { return frame_buffer_.dataV; } + + int StrideY() const override { return frame_buffer_.strideY; } + + int StrideU() const override { return frame_buffer_.strideU; } + + int StrideV() const override { return frame_buffer_.strideV; } + + private: + id frame_buffer_; + int width_; + int height_; +}; + +webrtc::scoped_refptr CreateNV12FromCVPixelBuffer( + CVPixelBufferRef pixelBuffer, + bool use_pool) { + // Only NV12 CVPixelBuffers can be wrapped/copied into an NV12Buffer. + if (!pixelBuffer) { + return nullptr; + } + + const OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (format != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange && + format != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + return nullptr; + } + + // Validate dimensions to avoid creating invalid buffers. + const int width = static_cast(CVPixelBufferGetWidth(pixelBuffer)); + const int height = static_cast(CVPixelBufferGetHeight(pixelBuffer)); + if (width <= 0 || height <= 0) { + return nullptr; + } + + webrtc::scoped_refptr nv12; + if (use_pool) { + // Thread-local pool avoids per-frame allocations on the hot path. + // The pool is intentionally leaked to avoid shutdown order issues. + thread_local webrtc::VideoFrameBufferPool* nv12_pool = + new webrtc::VideoFrameBufferPool(); + nv12 = nv12_pool->CreateNV12Buffer(width, height); + } else { + // Simple path: allocate a fresh NV12 buffer for this frame. + nv12 = webrtc::NV12Buffer::Create(width, height); + } + + if (!nv12) { + return nullptr; + } + + // Copy Y/UV planes into the new NV12 buffer. + CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + const uint8_t* src_y = static_cast( + CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); + const uint8_t* src_uv = static_cast( + CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); + if (!src_y || !src_uv) { + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + return nullptr; + } + const size_t src_stride_y = + CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); + const size_t src_stride_uv = + CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); + + uint8_t* dst_y = nv12->MutableDataY(); + uint8_t* dst_uv = nv12->MutableDataUV(); + const int dst_stride_y = nv12->StrideY(); + const int dst_stride_uv = nv12->StrideUV(); + + // NV12 expects full-width Y and UV rows. + const size_t y_row_bytes = static_cast(width); + const size_t uv_row_bytes = static_cast(width); + + // Copy each plane row-by-row to honor differing strides. + for (int row = 0; row < height; row++) { + memcpy(dst_y + row * dst_stride_y, + src_y + row * src_stride_y, + y_row_bytes); + } + + const int uv_rows = height / 2; + for (int row = 0; row < uv_rows; row++) { + memcpy(dst_uv + row * dst_stride_uv, + src_uv + row * src_stride_uv, + uv_row_bytes); + } + + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + return nv12; +} + +} // namespace + +// Wraps the ObjC buffer and snapshots its size. +ObjCFrameBuffer::ObjCFrameBuffer( + id frame_buffer) + : frame_buffer_(frame_buffer), + width_(frame_buffer.width), + height_(frame_buffer.height) {} + +ObjCFrameBuffer::~ObjCFrameBuffer() {} + +VideoFrameBuffer::Type ObjCFrameBuffer::type() const { + // Native means we own an ObjC-backed buffer wrapper. + return Type::kNative; +} + +int ObjCFrameBuffer::width() const { + return width_; +} + +int ObjCFrameBuffer::height() const { + return height_; +} + +webrtc::scoped_refptr ObjCFrameBuffer::ToI420() { + // Wrap the ObjC I420 buffer without copying. + return webrtc::make_ref_counted([frame_buffer_ toI420]); +} + +webrtc::scoped_refptr ObjCFrameBuffer::CropAndScale( + int offset_x, + int offset_y, + int crop_width, + int crop_height, + int scaled_width, + int scaled_height) { + // Prefer ObjC buffer's crop/scale if it implements it. + if ([frame_buffer_ + respondsToSelector:@selector + (cropAndScaleWith: + offsetY:cropWidth:cropHeight:scaleWidth:scaleHeight:)]) { + return webrtc::make_ref_counted([frame_buffer_ + cropAndScaleWith:offset_x + offsetY:offset_y + cropWidth:crop_width + cropHeight:crop_height + scaleWidth:scaled_width + scaleHeight:scaled_height]); + } + + // Fall back to WebRTC's default crop/scale path. + return VideoFrameBuffer::CropAndScale( + offset_x, offset_y, crop_width, crop_height, scaled_width, scaled_height); +} + +id ObjCFrameBuffer::wrapped_frame_buffer() + const { + // Expose the original ObjC buffer for native unwraps. + return frame_buffer_; +} + +id ToObjCVideoFrameBuffer( + const webrtc::scoped_refptr& buffer) { + // Preserve already-wrapped ObjC buffers to avoid extra bridging. + if (buffer->type() == VideoFrameBuffer::Type::kNative) { + id wrapped = + static_cast(buffer.get())->wrapped_frame_buffer(); + const auto policy = webrtc::ObjCGetFrameBufferPolicy(); + // Only convert CVPixelBuffer when policy explicitly requests it. + if ((policy == webrtc::ObjCFrameBufferPolicy::kCopyToNV12 || + policy == webrtc::ObjCFrameBufferPolicy::kConvertWithPoolToNV12) && + [wrapped isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *cv_buffer = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)wrapped; + const bool use_pool = + policy == webrtc::ObjCFrameBufferPolicy::kConvertWithPoolToNV12; + webrtc::scoped_refptr nv12_buffer = + CreateNV12FromCVPixelBuffer(cv_buffer.pixelBuffer, use_pool); + if (nv12_buffer) { + RTC_OBJC_TYPE(RTCNV12Buffer) *result = + [[RTC_OBJC_TYPE(RTCNV12Buffer) alloc] + initWithFrameBuffer:nv12_buffer]; + return result; + } + } + return wrapped; + } else if (buffer->type() == VideoFrameBuffer::Type::kNV12) { + // NV12 frames can be wrapped directly unless policy forbids NV12. + if (webrtc::ObjCGetFrameBufferPolicy() == + webrtc::ObjCFrameBufferPolicy::kNone) { + // Explicitly force I420 when policy is "none". + return [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] + initWithFrameBuffer:buffer->ToI420()]; + } + // GetNV12 returns a const pointer; underlying buffer is ref-counted. + webrtc::scoped_refptr nv12_buffer( + const_cast(buffer->GetNV12())); + // Wrap NV12 without copying for GPU-friendly consumption. + RTC_OBJC_TYPE(RTCNV12Buffer) *result = + [[RTC_OBJC_TYPE(RTCNV12Buffer) alloc] + initWithFrameBuffer:nv12_buffer]; + return result; + } else { + // Non-NV12 input; policy decides if we should convert to NV12 or keep I420. + const auto policy = webrtc::ObjCGetFrameBufferPolicy(); + if (policy == webrtc::ObjCFrameBufferPolicy::kNone || + policy == webrtc::ObjCFrameBufferPolicy::kWrapOnlyExistingNV12) { + // Either no NV12 allowed or we only wrap existing NV12 buffers. + return [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] + initWithFrameBuffer:buffer->ToI420()]; + } + // Convert to I420 first, then potentially to NV12. + webrtc::scoped_refptr i420_buffer = + buffer->ToI420(); + if (!i420_buffer) { + return nil; + } + + const int width = i420_buffer->width(); + const int height = i420_buffer->height(); + // Avoid conversion for very small frames to save overhead. + if (width * height < kMinNV12ConversionPixels) { + return [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] + initWithFrameBuffer:i420_buffer]; + } + + webrtc::scoped_refptr nv12_buffer; + if (policy == webrtc::ObjCFrameBufferPolicy::kCopyToNV12) { + // Simple path: let WebRTC allocate and copy. + nv12_buffer = webrtc::NV12Buffer::Copy(*i420_buffer); + } else { + // Pooled path: reuse NV12 buffers and convert via libyuv. + thread_local webrtc::VideoFrameBufferPool* nv12_pool = + new webrtc::VideoFrameBufferPool(); + webrtc::scoped_refptr pooled = + nv12_pool->CreateNV12Buffer(width, height); + if (pooled) { + // Convert planar I420 to bi-planar NV12 in-place. + libyuv::I420ToNV12(i420_buffer->DataY(), i420_buffer->StrideY(), + i420_buffer->DataU(), i420_buffer->StrideU(), + i420_buffer->DataV(), i420_buffer->StrideV(), + pooled->MutableDataY(), pooled->StrideY(), + pooled->MutableDataUV(), pooled->StrideUV(), + width, height); + nv12_buffer = pooled; + } + } + + if (!nv12_buffer) { + // If conversion failed, fall back to I420. + return [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] + initWithFrameBuffer:i420_buffer]; + } + // Wrap the converted NV12 buffer for ObjC consumption. + RTC_OBJC_TYPE(RTCNV12Buffer) *result = + [[RTC_OBJC_TYPE(RTCNV12Buffer) alloc] + initWithFrameBuffer:nv12_buffer]; + return result; + } +} + +} // namespace webrtc diff --git a/sdk/objc/native/src/objc_nv12_conversion.h b/sdk/objc/native/src/objc_nv12_conversion.h new file mode 100644 index 0000000000..e4d3d16e25 --- /dev/null +++ b/sdk/objc/native/src/objc_nv12_conversion.h @@ -0,0 +1,28 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_NV12_CONVERSION_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_NV12_CONVERSION_H_ + +namespace webrtc { + +enum class ObjCFrameBufferPolicy { + kNone = 0, + kWrapOnlyExistingNV12, + kCopyToNV12, + kConvertWithPoolToNV12 +}; + +ObjCFrameBufferPolicy ObjCGetFrameBufferPolicy(); +void SetObjCFrameBufferPolicy(ObjCFrameBufferPolicy policy); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_NV12_CONVERSION_H_ diff --git a/sdk/objc/native/src/objc_nv12_conversion.mm b/sdk/objc/native/src/objc_nv12_conversion.mm new file mode 100644 index 0000000000..7fab0d92fa --- /dev/null +++ b/sdk/objc/native/src/objc_nv12_conversion.mm @@ -0,0 +1,29 @@ +/* + * Copyright 2026 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_nv12_conversion.h" + +#include + +namespace webrtc { +namespace { +std::atomic g_frame_buffer_policy{ + ObjCFrameBufferPolicy::kNone}; +} // namespace + +ObjCFrameBufferPolicy ObjCGetFrameBufferPolicy() { + return g_frame_buffer_policy.load(std::memory_order_relaxed); +} + +void SetObjCFrameBufferPolicy(ObjCFrameBufferPolicy policy) { + g_frame_buffer_policy.store(policy, std::memory_order_relaxed); +} + +} // namespace webrtc diff --git a/webrtc.gni b/webrtc.gni index 60641a05e1..f5c5f80d12 100644 --- a/webrtc.gni +++ b/webrtc.gni @@ -69,6 +69,9 @@ declare_args() { # annotated symbols. rtc_enable_objc_symbol_export = rtc_enable_symbol_export + # Stream-specific toggle to enable shared rendering backend and NV12 policy. + stream_enable_rendering_backend = false + # Setting this to true will define WEBRTC_EXCLUDE_FIELD_TRIAL_DEFAULT which # will tell the pre-processor to remove the default definition of symbols # needed to use field_trial. In that case a new implementation needs to be