diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 53ecd57cf4..9a08834c6b 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1768,16 +1768,16 @@ PODS: - ReactCommon/turbomodule/core - Yoga - SocketRocket (0.7.0) - - VisionCamera (4.6.1): - - VisionCamera/Core (= 4.6.1) - - VisionCamera/FrameProcessors (= 4.6.1) - - VisionCamera/React (= 4.6.1) - - VisionCamera/Core (4.6.1) - - VisionCamera/FrameProcessors (4.6.1): + - VisionCamera (4.6.3): + - VisionCamera/Core (= 4.6.3) + - VisionCamera/FrameProcessors (= 4.6.3) + - VisionCamera/React (= 4.6.3) + - VisionCamera/Core (4.6.3) + - VisionCamera/FrameProcessors (4.6.3): - React - React-callinvoker - react-native-worklets-core - - VisionCamera/React (4.6.1): + - VisionCamera/React (4.6.3): - React-Core - VisionCamera/FrameProcessors - Yoga (0.0.0) @@ -2097,9 +2097,9 @@ SPEC CHECKSUMS: RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8 RNVectorIcons: 6382277afab3c54658e9d555ee0faa7a37827136 SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d - VisionCamera: ec141897a88c2e95e8b83cf97b8e4db801e02fd6 - Yoga: 055f92ad73f8c8600a93f0e25ac0b2344c3b07e6 + VisionCamera: 88df4dae7196c93ecd331f105f0e5d7d95702cb3 + Yoga: aa3df615739504eebb91925fc9c58b4922ea9a08 PODFILE CHECKSUM: 2ad84241179871ca890f7c65c855d117862f1a68 -COCOAPODS: 1.16.2 +COCOAPODS: 1.15.2 diff --git a/package/ios/Core/CameraConfiguration.swift b/package/ios/Core/CameraConfiguration.swift index a4b94dd377..6dea95508f 100644 --- a/package/ios/Core/CameraConfiguration.swift +++ b/package/ios/Core/CameraConfiguration.swift @@ -182,6 +182,7 @@ final class CameraConfiguration { var enableBufferCompression = false var enableHdr = false var enableFrameProcessor = false + var enableDepth = false } /** diff --git a/package/ios/Core/CameraSession+Configuration.swift b/package/ios/Core/CameraSession+Configuration.swift index ecd2a94daf..33dcc5dce4 100644 --- a/package/ios/Core/CameraSession+Configuration.swift +++ b/package/ios/Core/CameraSession+Configuration.swift @@ -54,7 +54,7 @@ extension CameraSession { // pragma MARK: Outputs /** - Configures all outputs (`photo` + `video` + `codeScanner`) + Configures all outputs (`photo` + `video` + `depth` + `codeScanner`) */ func configureOutputs(configuration: CameraConfiguration) throws { VisionLogger.log(level: .info, message: "Configuring Outputs...") @@ -65,7 +65,9 @@ extension CameraSession { } photoOutput = nil videoOutput = nil + depthOutput = nil codeScannerOutput = nil + outputSynchronizer = nil // Photo Output if case let .enabled(photo) = configuration.photo { @@ -97,7 +99,7 @@ extension CameraSession { } // Video Output + Frame Processor - if case .enabled = configuration.video { + if case let .enabled(video) = configuration.video { VisionLogger.log(level: .info, message: "Adding Video Data output...") // 1. Add @@ -107,8 +109,7 @@ extension CameraSession { } captureSession.addOutput(videoOutput) - // 2. Configure - videoOutput.setSampleBufferDelegate(self, queue: CameraQueues.videoQueue) + // 2. Configure Video videoOutput.alwaysDiscardsLateVideoFrames = true if configuration.isMirrored { // 2.1. If mirroring is enabled, mirror all connections along the vertical axis @@ -120,6 +121,24 @@ extension CameraSession { } } + // 3. Configure Depth + if video.enableDepth { + // Video is synchronized with depth data - use a joined delegate! + // 3.1. Create depth output + let depthOutput = AVCaptureDepthDataOutput() + depthOutput.alwaysDiscardsLateDepthData = true + depthOutput.isFilteringEnabled = false + // 3.2. Add depth output to session + captureSession.addOutput(depthOutput) + // 3.3. Set up a synchronizer between video and depth data + outputSynchronizer = AVCaptureDataOutputSynchronizer(dataOutputs: [depthOutput, videoOutput]) + outputSynchronizer!.setDelegate(self, queue: CameraQueues.videoQueue) + self.depthOutput = depthOutput + } else { + // Video is the only output - use it's own delegate + videoOutput.setSampleBufferDelegate(self, queue: CameraQueues.videoQueue) + } + self.videoOutput = videoOutput } diff --git a/package/ios/Core/CameraSession.swift b/package/ios/Core/CameraSession.swift index 10b0f3399c..bfea30b623 100644 --- a/package/ios/Core/CameraSession.swift +++ b/package/ios/Core/CameraSession.swift @@ -13,7 +13,11 @@ import Foundation A fully-featured Camera Session supporting preview, video, photo, frame processing, and code scanning outputs. All changes to the session have to be controlled via the `configure` function. */ -final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate { +final class CameraSession: + NSObject, + AVCaptureVideoDataOutputSampleBufferDelegate, + AVCaptureAudioDataOutputSampleBufferDelegate, + AVCaptureDataOutputSynchronizerDelegate { // Configuration private var isInitialized = false var configuration: CameraConfiguration? @@ -27,7 +31,9 @@ final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat var photoOutput: AVCapturePhotoOutput? var videoOutput: AVCaptureVideoDataOutput? var audioOutput: AVCaptureAudioDataOutput? + var depthOutput: AVCaptureDepthDataOutput? var codeScannerOutput: AVCaptureMetadataOutput? + var outputSynchronizer: AVCaptureDataOutputSynchronizer? // State var metadataProvider = MetadataProvider() var recordingSession: RecordingSession? @@ -276,7 +282,24 @@ final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat } } - private final func onVideoFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool) { + func dataOutputSynchronizer(_: AVCaptureDataOutputSynchronizer, didOutput synchronizedDataCollection: AVCaptureSynchronizedDataCollection) { + guard let videoOutput else { return } + guard let videoData = synchronizedDataCollection.synchronizedData(for: videoOutput) as? AVCaptureSynchronizedSampleBufferData else { return } + + if let depthOutput { + // We have depth data as well + guard let depthData = synchronizedDataCollection.synchronizedData(for: videoOutput) as? AVCaptureSynchronizedSampleBufferData else { return } + onVideoFrame(sampleBuffer: videoData.sampleBuffer, + orientation: videoOutput.orientation, + isMirrored: videoOutput.isMirrored, + depthData: depthData.sampleBuffer) + } else { + // We only have video data + onVideoFrame(sampleBuffer: videoData.sampleBuffer, orientation: videoOutput.orientation, isMirrored: videoOutput.isMirrored) + } + } + + private final func onVideoFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthData: CMSampleBuffer? = nil) { if let recordingSession { do { // Write the Video Buffer to the .mov/.mp4 file @@ -290,7 +313,10 @@ final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat if let delegate { // Call Frame Processor (delegate) for every Video Frame - delegate.onFrame(sampleBuffer: sampleBuffer, orientation: orientation, isMirrored: isMirrored) + delegate.onFrame(sampleBuffer: sampleBuffer, + orientation: orientation, + isMirrored: isMirrored, + depthBuffer: depthData) } } diff --git a/package/ios/Core/CameraSessionDelegate.swift b/package/ios/Core/CameraSessionDelegate.swift index 07ec0a7410..d73830fed2 100644 --- a/package/ios/Core/CameraSessionDelegate.swift +++ b/package/ios/Core/CameraSessionDelegate.swift @@ -44,7 +44,7 @@ protocol CameraSessionDelegate: AnyObject { /** Called for every frame (if video or frameProcessor is enabled) */ - func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool) + func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthBuffer: CMSampleBuffer?) /** Called whenever a QR/Barcode has been scanned. Only if the CodeScanner Output is enabled */ diff --git a/package/ios/FrameProcessors/Frame.h b/package/ios/FrameProcessors/Frame.h index a925b8813a..2cacdbd9e3 100644 --- a/package/ios/FrameProcessors/Frame.h +++ b/package/ios/FrameProcessors/Frame.h @@ -16,13 +16,17 @@ NS_ASSUME_NONNULL_BEGIN @interface Frame : NSObject -- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored; +- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer + orientation:(UIImageOrientation)orientation + isMirrored:(BOOL)isMirrored + depthData:(nullable CMSampleBufferRef)depth; - (instancetype)init NS_UNAVAILABLE; - (void)incrementRefCount; - (void)decrementRefCount; @property(nonatomic, readonly) CMSampleBufferRef buffer; +@property(nonatomic, readonly, nullable) CMSampleBufferRef depth; @property(nonatomic, readonly) UIImageOrientation orientation; @property(nonatomic, readonly) NSString* pixelFormat; diff --git a/package/ios/FrameProcessors/Frame.m b/package/ios/FrameProcessors/Frame.m index 663e19f4f7..08e828d54c 100644 --- a/package/ios/FrameProcessors/Frame.m +++ b/package/ios/FrameProcessors/Frame.m @@ -14,14 +14,19 @@ @implementation Frame { CMSampleBufferRef _Nonnull _buffer; UIImageOrientation _orientation; BOOL _isMirrored; + CMSampleBufferRef _Nullable _depth; } -- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored { +- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer + orientation:(UIImageOrientation)orientation + isMirrored:(BOOL)isMirrored + depthData:(nullable CMSampleBufferRef)depth { self = [super init]; if (self) { _buffer = buffer; _orientation = orientation; _isMirrored = isMirrored; + _depth = depth; } return self; } @@ -47,6 +52,10 @@ - (CMSampleBufferRef)buffer { return _buffer; } +- (nullable CMSampleBufferRef)depth { + return _depth; +} + - (BOOL)isValid { return _buffer != nil && CFGetRetainCount(_buffer) > 0 && CMSampleBufferIsValid(_buffer); } diff --git a/package/ios/FrameProcessors/FrameHostObject.mm b/package/ios/FrameProcessors/FrameHostObject.mm index 67bbfe51df..4b8cdd0068 100644 --- a/package/ios/FrameProcessors/FrameHostObject.mm +++ b/package/ios/FrameProcessors/FrameHostObject.mm @@ -74,6 +74,16 @@ if (name == "planesCount") { return jsi::Value((double)_frame.planesCount); } + if (name == "depth") { + if (_frame.depth == nil) { + return jsi::Value::undefined(); + } + jsi::Object object(runtime); + CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_frame.depth); + object.setProperty(runtime, "width", jsi::Value(static_cast(CVPixelBufferGetWidth(imageBuffer)))); + object.setProperty(runtime, "height", jsi::Value(static_cast(CVPixelBufferGetHeight(imageBuffer)))); + return object; + } // Internal methods if (name == "incrementRefCount") { diff --git a/package/ios/React/CameraView.swift b/package/ios/React/CameraView.swift index c773975353..8f885c4e9f 100644 --- a/package/ios/React/CameraView.swift +++ b/package/ios/React/CameraView.swift @@ -211,7 +211,8 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat config.video = .enabled(config: CameraConfiguration.Video(pixelFormat: getPixelFormat(), enableBufferCompression: enableBufferCompression, enableHdr: videoHdr, - enableFrameProcessor: enableFrameProcessor)) + enableFrameProcessor: enableFrameProcessor, + enableDepth: enableDepthData)) } else { config.video = .disabled } @@ -362,7 +363,7 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat ]) } - func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool) { + func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthBuffer: CMSampleBuffer?) { // Update latest frame that can be used for snapshot capture latestVideoFrame = Snapshot(imageBuffer: sampleBuffer, orientation: orientation) @@ -374,7 +375,8 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat // Call Frame Processor let frame = Frame(buffer: sampleBuffer, orientation: orientation.imageOrientation, - isMirrored: isMirrored) + isMirrored: isMirrored, + depthData: depthBuffer) frameProcessor.call(frame) } #endif diff --git a/package/src/devices/getCameraFormat.ts b/package/src/devices/getCameraFormat.ts index ec48e951b8..d6e72316f0 100644 --- a/package/src/devices/getCameraFormat.ts +++ b/package/src/devices/getCameraFormat.ts @@ -74,6 +74,10 @@ export interface FormatFilter { * you might want to choose a different auto-focus system. */ autoFocusSystem?: AutoFocusSystem + /** + * Specifies whether to prefer formats that support depth data capture. + */ + depth?: boolean } type FilterWithPriority = { @@ -236,6 +240,12 @@ export function getCameraFormat(device: CameraDevice, filters: FormatFilter[]): if (format.autoFocusSystem === filter.autoFocusSystem.target) rightPoints += filter.autoFocusSystem.priority } + // Find depth data + if (filter.depth != null) { + if (bestFormat.supportsDepthCapture) leftPoints += filter.depth.priority + if (format.supportsDepthCapture) rightPoints += filter.depth.priority + } + if (rightPoints > leftPoints) bestFormat = format } diff --git a/package/src/types/Frame.ts b/package/src/types/Frame.ts index c9e6bb2310..f5aba3ca8e 100644 --- a/package/src/types/Frame.ts +++ b/package/src/types/Frame.ts @@ -64,6 +64,14 @@ export interface Frame { */ readonly pixelFormat: PixelFormat + /** + * Represents the depth data of this Frame, if the Camera is configured to stream depth data. + */ + readonly depth?: { + readonly width: number + readonly height: number + } + /** * Get the underlying data of the Frame as a uint8 array buffer. *