From 2e2d9b4bbb5b34c3888d9e08331fc2628ed7ddb4 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 1 May 2024 19:11:19 +0200 Subject: [PATCH] feat: Completely refactor the `FrameProcessorPlugins.ts` file into multiple files (#2830) * feat: Completely refactor the `FrameProcessorPlugins.ts` file into multiple files * Refactor * Prepare * Update some docs * fix: Fix invalid `options` param (undefined) * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx --- .../FRAME_PROCESSORS_CREATE_OVERVIEW.mdx | 2 +- .../guides/FRAME_PROCESSOR_CREATE_FINAL.mdx | 4 +- .../FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx | 4 +- .../FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx | 2 +- .../ExampleKotlinSwiftPlugin.ts | 4 +- .../src/frame-processors/ExamplePlugin.ts | 4 +- package/src/Camera.tsx | 2 +- package/src/FrameProcessorPlugins.ts | 291 ------------------ .../FrameProcessorsUnavailableError.ts | 11 + .../src/frame-processors/VisionCameraProxy.ts | 70 +++++ .../initFrameProcessorPlugin.ts | 34 ++ package/src/frame-processors/runAsync.ts | 100 ++++++ .../src/frame-processors/runAtTargetFps.ts | 58 ++++ .../src/frame-processors/throwErrorOnJS.ts | 51 +++ .../frame-processors/withFrameRefCounting.ts | 26 ++ package/src/hooks/useFrameProcessor.ts | 4 +- package/src/index.ts | 13 +- package/src/skia/useSkiaFrameProcessor.ts | 5 +- 18 files changed, 378 insertions(+), 307 deletions(-) delete mode 100644 package/src/FrameProcessorPlugins.ts create mode 100644 package/src/frame-processors/FrameProcessorsUnavailableError.ts create mode 100644 package/src/frame-processors/VisionCameraProxy.ts create mode 100644 package/src/frame-processors/initFrameProcessorPlugin.ts create mode 100644 package/src/frame-processors/runAsync.ts create mode 100644 package/src/frame-processors/runAtTargetFps.ts create mode 100644 package/src/frame-processors/throwErrorOnJS.ts create mode 100644 package/src/frame-processors/withFrameRefCounting.ts diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index 23428c9afd..ef8e77964a 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -64,7 +64,7 @@ Returns a `string` in JS: ```js export function detectObject(frame: Frame): string { 'worklet' - const result = FrameProcessorPlugins.detectObject(frame) + const result = detectObject(frame) console.log(result) // <-- "cat" } ``` diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx index b60a235e5a..b8add9aa8e 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx @@ -9,9 +9,9 @@ sidebar_label: Finish creating your Frame Processor Plugin To make the Frame Processor Plugin available to the Frame Processor Worklet Runtime, create the following wrapper function in JS/TS: ```ts -import { VisionCameraProxy, Frame } from 'react-native-vision-camera' +import { initFrameProcessorPlugin, Frame } from 'react-native-vision-camera' -const plugin = VisionCameraProxy.initFrameProcessorPlugin('scanFaces') +const plugin = initFrameProcessorPlugin('scanFaces') /** * Scans faces. diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx index 8669a947e4..15b0bdf3da 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx @@ -109,7 +109,7 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { ``` :::note -The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy` object. In this case, it would be `VisionCameraProxy.initFrameProcessorPlugin("detectFaces")`. +The Frame Processor Plugin can be initialized from JavaScript using `initFrameProcessorPlugin("detectFaces")`. ::: 6. Register the package in MainApplication.java @@ -181,7 +181,7 @@ class FaceDetectorFrameProcessorPluginPackage : ReactPackage { ``` :::note -The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy` object. In this case, it would be `VisionCameraProxy.initFrameProcessorPlugin("detectFaces")`. +The Frame Processor Plugin can be initialized from JavaScript using `initFrameProcessorPlugin("detectFaces")`. ::: 6. Register the package in MainApplication.java diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx index 33800034f9..75b0628dd7 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx @@ -72,7 +72,7 @@ VISION_EXPORT_FRAME_PROCESSOR(FaceDetectorFrameProcessorPlugin, detectFaces) ``` :::note -The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy` object. In this case, it would be `VisionCameraProxy.initFrameProcessorPlugin("detectFaces")`. +The Frame Processor Plugin can be initialized from JavaScript using `initFrameProcessorPlugin("detectFaces")`. ::: 4. **Implement your Frame Processing.** See the [Example Plugin (Objective-C)](https://github.com/mrousavy/react-native-vision-camera/blob/main/package/example/ios/FrameProcessors%20Plugins/Example%20Plugin/ExampleFrameProcessorPlugin.m) for reference. diff --git a/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts b/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts index 416e7fda1d..c02e758445 100644 --- a/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts +++ b/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts @@ -1,7 +1,7 @@ import type { Frame } from 'react-native-vision-camera' -import { VisionCameraProxy } from 'react-native-vision-camera' +import { initFrameProcessorPlugin } from 'react-native-vision-camera' -const plugin = VisionCameraProxy.initFrameProcessorPlugin('example_kotlin_swift_plugin', { foo: 'bar' }) +const plugin = initFrameProcessorPlugin('example_kotlin_swift_plugin', { foo: 'bar' }) export function exampleKotlinSwiftPlugin(frame: Frame): string[] { 'worklet' diff --git a/package/example/src/frame-processors/ExamplePlugin.ts b/package/example/src/frame-processors/ExamplePlugin.ts index 4f705cbbe3..7810464e41 100644 --- a/package/example/src/frame-processors/ExamplePlugin.ts +++ b/package/example/src/frame-processors/ExamplePlugin.ts @@ -1,7 +1,7 @@ import type { Frame } from 'react-native-vision-camera' -import { VisionCameraProxy } from 'react-native-vision-camera' +import { initFrameProcessorPlugin } from 'react-native-vision-camera' -const plugin = VisionCameraProxy.initFrameProcessorPlugin('example_plugin') +const plugin = initFrameProcessorPlugin('example_plugin') interface Result { example_array: (string | number | boolean)[] diff --git a/package/src/Camera.tsx b/package/src/Camera.tsx index a2869decc9..6113515cac 100644 --- a/package/src/Camera.tsx +++ b/package/src/Camera.tsx @@ -8,7 +8,7 @@ import { CameraModule } from './NativeCameraModule' import type { PhotoFile, TakePhotoOptions } from './types/PhotoFile' import type { Point } from './types/Point' import type { RecordVideoOptions, VideoFile } from './types/VideoFile' -import { VisionCameraProxy } from './FrameProcessorPlugins' +import { VisionCameraProxy } from './frame-processors/VisionCameraProxy' import { CameraDevices } from './CameraDevices' import type { EmitterSubscription, NativeSyntheticEvent, NativeMethods } from 'react-native' import type { TakeSnapshotOptions } from './types/Snapshot' diff --git a/package/src/FrameProcessorPlugins.ts b/package/src/FrameProcessorPlugins.ts deleted file mode 100644 index f22d275754..0000000000 --- a/package/src/FrameProcessorPlugins.ts +++ /dev/null @@ -1,291 +0,0 @@ -import type { Frame, FrameInternal } from './types/Frame' -import { CameraRuntimeError } from './CameraError' -import { CameraModule } from './NativeCameraModule' -import { assertJSIAvailable } from './JSIHelper' -import { WorkletsProxy } from './dependencies/WorkletsProxy' -import type { IWorkletContext } from 'react-native-worklets-core' - -declare global { - // eslint-disable-next-line no-var - var __frameProcessorRunAtTargetFpsMap: Record | undefined - // eslint-disable-next-line no-var - var __ErrorUtils: - | { - reportFatalError: (error: unknown) => void - } - | undefined -} - -type BasicParameterType = string | number | boolean | undefined -type ParameterType = BasicParameterType | BasicParameterType[] | Record - -/** - * An initialized native instance of a FrameProcessorPlugin. - * All memory allocated by this plugin will be deleted once this value goes out of scope. - */ -export interface FrameProcessorPlugin { - /** - * Call the native Frame Processor Plugin with the given Frame and options. - * @param frame The Frame from the Frame Processor. - * @param options (optional) Additional options. Options will be converted to a native dictionary - * @returns (optional) A value returned from the native Frame Processor Plugin (or undefined) - */ - call(frame: Frame, options?: Record): ParameterType -} - -interface TVisionCameraProxy { - /** - * @internal - */ - setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void): void - /** - * @internal - */ - removeFrameProcessor(viewTag: number): void - /** - * Creates a new instance of a native Frame Processor Plugin. - * The Plugin has to be registered on the native side, otherwise this returns `undefined`. - * @param name The name of the Frame Processor Plugin. This has to be the same name as on the native side. - * @param options (optional) Options, as a native dictionary, passed to the constructor/init-function of the native plugin. - * @example - * ```ts - * const plugin = VisionCameraProxy.initFrameProcessorPlugin('scanFaces', { model: 'fast' }) - * if (plugin == null) throw new Error("Failed to load scanFaces plugin!") - * ``` - */ - initFrameProcessorPlugin(name: string, options?: Record): FrameProcessorPlugin | undefined - /** - * Throws the given error. - */ - throwJSError(error: unknown): void - /** - * Get the Frame Processor Runtime Worklet Context. - * - * This is the serial [DispatchQueue](https://developer.apple.com/documentation/dispatch/dispatchqueue) - * / [Executor](https://developer.android.com/reference/java/util/concurrent/Executor) the - * video/frame processor pipeline is running on. - */ - workletContext: IWorkletContext | undefined -} - -const errorMessage = 'Frame Processors are not available, react-native-worklets-core is not installed!' - -let hasWorklets = false -let isAsyncContextBusy = { value: false } -let runOnAsyncContext = (_frame: Frame, _func: () => void): void => { - throw new CameraRuntimeError('system/frame-processors-unavailable', errorMessage) -} -let throwJSError = (error: unknown): void => { - throw error -} - -try { - assertJSIAvailable() - - // this will lazyily require the react-native-worklets-core dependency. If this throws, worklets are not available. - const Worklets = WorkletsProxy.Worklets - - // fill the stubs with the actual native Worklets implementations - const throwErrorOnJS = Worklets.createRunOnJS((message: string, stack: string | undefined) => { - const error = new Error() - error.message = message - error.stack = stack - error.name = 'Frame Processor Error' - // @ts-expect-error this is react-native specific - error.jsEngine = 'VisionCamera' - - // From react-native: - // @ts-expect-error ErrorUtils is in global. - if (global.ErrorUtils != null) - // @ts-expect-error the reportFatalError method is an internal method of ErrorUtils not exposed in the type definitions - global.ErrorUtils.reportFatalError(error) - else if (global.__ErrorUtils != null) global.__ErrorUtils.reportFatalError(error) - else console.error('Frame Processor Error:', error) - }) - throwJSError = (error) => { - 'worklet' - const safeError = error as Error | undefined - const message = safeError != null && 'message' in safeError ? safeError.message : 'Frame Processor threw an error.' - throwErrorOnJS(message, safeError?.stack) - } - - isAsyncContextBusy = Worklets.createSharedValue(false) - const asyncContext = Worklets.createContext('VisionCamera.async') - runOnAsyncContext = asyncContext.createRunAsync((frame: Frame, func: () => void) => { - 'worklet' - try { - // Call long-running function - func() - } catch (e) { - // Re-throw error on JS Thread - throwJSError(e) - } finally { - // Potentially delete Frame if we were the last ref - const internal = frame as FrameInternal - internal.decrementRefCount() - - isAsyncContextBusy.value = false - } - }) - hasWorklets = true -} catch (e) { - // Worklets are not installed, so Frame Processors are disabled. -} - -let proxy: TVisionCameraProxy = { - initFrameProcessorPlugin: () => { - throw new CameraRuntimeError('system/frame-processors-unavailable', errorMessage) - }, - removeFrameProcessor: () => { - throw new CameraRuntimeError('system/frame-processors-unavailable', errorMessage) - }, - setFrameProcessor: () => { - throw new CameraRuntimeError('system/frame-processors-unavailable', errorMessage) - }, - throwJSError: throwJSError, - workletContext: undefined, -} -if (hasWorklets) { - // Install native Frame Processor Runtime Manager - const result = CameraModule.installFrameProcessorBindings() as unknown - if (result !== true) - throw new CameraRuntimeError('system/frame-processors-unavailable', 'Failed to install Frame Processor JSI bindings!') - - // @ts-expect-error global is untyped, it's a C++ host-object - proxy = global.VisionCameraProxy as TVisionCameraProxy - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (proxy == null) { - throw new CameraRuntimeError( - 'system/frame-processors-unavailable', - 'Failed to install VisionCameraProxy. Are Frame Processors properly enabled?', - ) - } -} - -export const VisionCameraProxy: TVisionCameraProxy = { - initFrameProcessorPlugin: proxy.initFrameProcessorPlugin, - removeFrameProcessor: proxy.removeFrameProcessor, - setFrameProcessor: proxy.setFrameProcessor, - throwJSError: throwJSError, - workletContext: proxy.workletContext, -} - -function getLastFrameProcessorCall(frameProcessorFuncId: string): number { - 'worklet' - return global.__frameProcessorRunAtTargetFpsMap?.[frameProcessorFuncId] ?? 0 -} -function setLastFrameProcessorCall(frameProcessorFuncId: string, value: number): void { - 'worklet' - if (global.__frameProcessorRunAtTargetFpsMap == null) global.__frameProcessorRunAtTargetFpsMap = {} - global.__frameProcessorRunAtTargetFpsMap[frameProcessorFuncId] = value -} - -/** - * Runs the given function at the given target FPS rate. - * - * For example, if you want to run a heavy face detection algorithm - * only once per second, you can use `runAtTargetFps(1, ...)` to - * throttle it to 1 FPS. - * - * @param fps The target FPS rate at which the given function should be executed - * @param func The function to execute. - * @returns The result of the function if it was executed, or `undefined` otherwise. - * @worklet - * @example - * - * ```ts - * const frameProcessor = useFrameProcessor((frame) => { - * 'worklet' - * console.log('New Frame') - * runAtTargetFps(5, () => { - * 'worklet' - * const faces = detectFaces(frame) - * console.log(`Detected a new face: ${faces[0]}`) - * }) - * }) - * ``` - */ -export function runAtTargetFps(fps: number, func: () => T): T | undefined { - 'worklet' - // @ts-expect-error - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const funcId = func.__workletHash ?? '1' - - const targetIntervalMs = 1000 / fps // <-- 60 FPS => 16,6667ms interval - const now = performance.now() - const diffToLastCall = now - getLastFrameProcessorCall(funcId) - if (diffToLastCall >= targetIntervalMs) { - setLastFrameProcessorCall(funcId, now) - // Last Frame Processor call is already so long ago that we want to make a new call - return func() - } - return undefined -} - -/** - * Runs the given function asynchronously, while keeping a strong reference to the Frame. - * - * For example, if you want to run a heavy face detection algorithm - * while still drawing to the screen at 60 FPS, you can use `runAsync(...)` - * to offload the face detection algorithm to a separate thread. - * - * @param frame The current Frame of the Frame Processor. - * @param func The function to execute. - * @worklet - * @example - * - * ```ts - * const frameProcessor = useFrameProcessor((frame) => { - * 'worklet' - * console.log('New Frame') - * runAsync(frame, () => { - * 'worklet' - * const faces = detectFaces(frame) - * const face = [faces0] - * console.log(`Detected a new face: ${face}`) - * }) - * }) - * ``` - */ -export function runAsync(frame: Frame, func: () => void): void { - 'worklet' - - if (isAsyncContextBusy.value) { - // async context is currently busy, we cannot schedule new work in time. - // drop this frame/runAsync call. - return - } - - // Increment ref count by one - const internal = frame as FrameInternal - internal.incrementRefCount() - - isAsyncContextBusy.value = true - - // Call in separate background context - runOnAsyncContext(frame, func) -} - -/** - * A private API to wrap a Frame Processor with a ref-counting mechanism - * @worklet - * @internal - */ -export function wrapFrameProcessorWithRefCounting(frameProcessor: (frame: Frame) => void): (frame: Frame) => void { - return (frame) => { - 'worklet' - // Increment ref-count by one - const internal = frame as FrameInternal - internal.incrementRefCount() - try { - // Call sync frame processor - frameProcessor(frame) - } catch (e) { - // Re-throw error on JS Thread - VisionCameraProxy.throwJSError(e) - } finally { - // Potentially delete Frame if we were the last ref (no runAsync) - internal.decrementRefCount() - } - } -} diff --git a/package/src/frame-processors/FrameProcessorsUnavailableError.ts b/package/src/frame-processors/FrameProcessorsUnavailableError.ts new file mode 100644 index 0000000000..02b70e9edb --- /dev/null +++ b/package/src/frame-processors/FrameProcessorsUnavailableError.ts @@ -0,0 +1,11 @@ +import { CameraRuntimeError } from '../CameraError' + +export class FrameProcessorsUnavailableError extends CameraRuntimeError { + constructor(reason: unknown) { + super( + 'system/frame-processors-unavailable', + 'Frame Processors are not available, react-native-worklets-core is not installed! ' + + `Error: ${reason instanceof Error ? reason.message : reason}`, + ) + } +} diff --git a/package/src/frame-processors/VisionCameraProxy.ts b/package/src/frame-processors/VisionCameraProxy.ts new file mode 100644 index 0000000000..8bf107508b --- /dev/null +++ b/package/src/frame-processors/VisionCameraProxy.ts @@ -0,0 +1,70 @@ +import type { IWorkletContext } from 'react-native-worklets-core' +import { CameraModule } from '../NativeCameraModule' +import type { Frame } from '../types/Frame' +import { FrameProcessorsUnavailableError } from './FrameProcessorsUnavailableError' +import type { FrameProcessorPlugin, ParameterType } from './initFrameProcessorPlugin' + +interface TVisionCameraProxy { + /** + * @internal + */ + setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void): void + /** + * @internal + */ + removeFrameProcessor(viewTag: number): void + /** + * @internal + * @deprecated + */ + initFrameProcessorPlugin(name: string, options: Record): FrameProcessorPlugin | undefined + /** + * Get the Frame Processor Runtime Worklet Context. + * + * This is the serial [DispatchQueue](https://developer.apple.com/documentation/dispatch/dispatchqueue) + * / [Executor](https://developer.android.com/reference/java/util/concurrent/Executor) the + * video/frame processor pipeline is running on. + * + * @internal + */ + workletContext: IWorkletContext | undefined +} + +let proxy: TVisionCameraProxy + +try { + // 1. Load react-native-worklets-core + require('react-native-worklets-core') + // 2. If react-native-worklets-core could be loaded, try to install Frame Processor bindings + const result = CameraModule.installFrameProcessorBindings() as unknown + if (result !== true) throw new Error(`Failed to install Frame Processor JSI bindings! installFrameProcessorBindings() returned ${result}`) + + // 3. Get global.VisionCameraProxy which was just installed by installFrameProcessorBindings() + // @ts-expect-error it's a global JSI variable injected by native + const globalProxy = global.VisionCameraProxy as TVisionCameraProxy | undefined + if (globalProxy == null) throw new Error('global.VisionCameraProxy is not installed! Was installFrameProcessorBindings() called?') + + proxy = globalProxy +} catch (e) { + // global.VisionCameraProxy is not injected! + // Just use dummy implementations that will throw when the user tries to use Frame Processors. + proxy = { + initFrameProcessorPlugin: () => { + throw new FrameProcessorsUnavailableError(e) + }, + removeFrameProcessor: () => { + throw new FrameProcessorsUnavailableError(e) + }, + setFrameProcessor: () => { + throw new FrameProcessorsUnavailableError(e) + }, + workletContext: undefined, + } +} + +/** + * The JSI Proxy for the Frame Processors Runtime. + * + * This will be replaced with a CxxTurboModule in the future. + */ +export const VisionCameraProxy = proxy diff --git a/package/src/frame-processors/initFrameProcessorPlugin.ts b/package/src/frame-processors/initFrameProcessorPlugin.ts new file mode 100644 index 0000000000..e2c03e396f --- /dev/null +++ b/package/src/frame-processors/initFrameProcessorPlugin.ts @@ -0,0 +1,34 @@ +import type { Frame } from '../types/Frame' +import { VisionCameraProxy } from './VisionCameraProxy' + +type BasicParameterType = string | number | boolean | undefined +export type ParameterType = BasicParameterType | BasicParameterType[] | Record + +/** + * An initialized native instance of a FrameProcessorPlugin. + * All memory allocated by this plugin will be deleted once this value goes out of scope. + */ +export interface FrameProcessorPlugin { + /** + * Call the native Frame Processor Plugin with the given Frame and options. + * @param frame The Frame from the Frame Processor. + * @param options (optional) Additional options. Options will be converted to a native dictionary + * @returns (optional) A value returned from the native Frame Processor Plugin (or undefined) + */ + call(frame: Frame, options?: Record): ParameterType +} + +/** + * Creates a new instance of a native Frame Processor Plugin. + * The Plugin has to be registered on the native side, otherwise this returns `undefined`. + * @param name The name of the Frame Processor Plugin. This has to be the same name as on the native side. + * @param options (optional) Options, as a native dictionary, passed to the constructor/init-function of the native plugin. + * @example + * ```ts + * const plugin = VisionCameraProxy.initFrameProcessorPlugin('scanFaces', { model: 'fast' }) + * if (plugin == null) throw new Error("Failed to load scanFaces plugin!") + * ``` + */ +export function initFrameProcessorPlugin(name: string, options: Record = {}): FrameProcessorPlugin | undefined { + return VisionCameraProxy.initFrameProcessorPlugin(name, options) +} diff --git a/package/src/frame-processors/runAsync.ts b/package/src/frame-processors/runAsync.ts new file mode 100644 index 0000000000..c27ed78f71 --- /dev/null +++ b/package/src/frame-processors/runAsync.ts @@ -0,0 +1,100 @@ +import { WorkletsProxy } from '../dependencies/WorkletsProxy' +import type { Frame, FrameInternal } from '../types/Frame' +import { FrameProcessorsUnavailableError } from './FrameProcessorsUnavailableError' +import { throwErrorOnJS } from './throwErrorOnJS' + +/** + * A synchronized Shared Value to indicate whether the async context is currently executing + */ +let isAsyncContextBusy: { value: boolean } +/** + * Runs the given function on the async context, and sets {@linkcode isAsyncContextBusy} to false after it finished executing. + */ +let runOnAsyncContext: (frame: Frame, func: () => void) => void + +try { + const Worklets = WorkletsProxy.Worklets + isAsyncContextBusy = Worklets.createSharedValue(false) + + const asyncContext = Worklets.createContext('VisionCamera.async') + runOnAsyncContext = asyncContext.createRunAsync((frame: Frame, func: () => void) => { + 'worklet' + try { + // Call long-running function + func() + } catch (e) { + // Re-throw error on JS Thread + throwErrorOnJS(e) + } finally { + // Potentially delete Frame if we were the last ref + const internal = frame as FrameInternal + internal.decrementRefCount() + + // free up async context again, new calls can be made + isAsyncContextBusy.value = false + } + }) +} catch (e) { + // react-native-worklets-core is not installed! + // Just use dummy implementations that will throw when the user tries to use Frame Processors. + isAsyncContextBusy = { value: false } + runOnAsyncContext = () => { + throw new FrameProcessorsUnavailableError(e) + } +} + +/** + * Runs the given {@linkcode func} asynchronously on a separate thread, + * allowing the Frame Processor to continue executing without dropping a Frame. + * + * Only one {@linkcode runAsync} call will execute at the same time, + * so {@linkcode runAsync} is **not parallel**, **but asynchronous**. + * + * + * For example, if your Camera is running at 60 FPS (16ms per frame), and a + * heavy ML face detection Frame Processor Plugin takes 500ms to execute, + * you have two options: + * - Run the plugin normally (synchronously in `useFrameProcessor`) + * but drop a lot of Frames, as we can only run at 2 FPS (500ms per frame) + * - Call the plugin inside {@linkcode runAsync} to allow the Camera to still + * run at 60 FPS, but offload the heavy ML face detection plugin to the + * asynchronous context, where it will run at 2 FPS. + * + * @note {@linkcode runAsync} cannot be used to draw to a Frame in a Skia Frame Processor. + * @param frame The current Frame of the Frame Processor. + * @param func The function to execute. + * @worklet + * @example + * + * ```ts + * const frameProcessor = useFrameProcessor((frame) => { + * 'worklet' + * console.log('New Frame arrived!') + * + * runAsync(frame, () => { + * 'worklet' + * const faces = detectFaces(frame) + * const face = [faces0] + * console.log(`Detected a new face: ${face}`) + * }) + * }) + * ``` + */ +export function runAsync(frame: Frame, func: () => void): void { + 'worklet' + + if (isAsyncContextBusy.value) { + // async context is currently busy, we cannot schedule new work in time. + // drop this frame/runAsync call. + return + } + + // Increment ref count by one + const internal = frame as FrameInternal + internal.incrementRefCount() + + isAsyncContextBusy.value = true + + // Call in separate background context + runOnAsyncContext(frame, func) +} diff --git a/package/src/frame-processors/runAtTargetFps.ts b/package/src/frame-processors/runAtTargetFps.ts new file mode 100644 index 0000000000..757f00be89 --- /dev/null +++ b/package/src/frame-processors/runAtTargetFps.ts @@ -0,0 +1,58 @@ +declare global { + // eslint-disable-next-line no-var + var __frameProcessorRunAtTargetFpsMap: Record | undefined +} + +function getLastFrameProcessorCall(frameProcessorFuncId: string): number { + 'worklet' + return global.__frameProcessorRunAtTargetFpsMap?.[frameProcessorFuncId] ?? 0 +} +function setLastFrameProcessorCall(frameProcessorFuncId: string, value: number): void { + 'worklet' + if (global.__frameProcessorRunAtTargetFpsMap == null) global.__frameProcessorRunAtTargetFpsMap = {} + global.__frameProcessorRunAtTargetFpsMap[frameProcessorFuncId] = value +} + +/** + * Runs the given {@linkcode func} at the given target {@linkcode fps} rate. + * + * {@linkcode runAtTargetFps} still executes the given {@linkcode func} synchronously, + * so this is only useful for throttling calls to a plugin or logger. + * + * For example, if you want to scan faces only once per second to avoid excessive + * CPU usage, use {@linkcode runAtTargetFps runAtTargetFps(1, ...)}. + * + * @param fps The target FPS rate at which the given function should be executed + * @param func The function to execute. + * @returns The result of the function if it was executed, or `undefined` otherwise. + * @worklet + * @example + * + * ```ts + * const frameProcessor = useFrameProcessor((frame) => { + * 'worklet' + * console.log('New Frame') + * runAtTargetFps(5, () => { + * 'worklet' + * const faces = detectFaces(frame) + * console.log(`Detected a new face: ${faces[0]}`) + * }) + * }) + * ``` + */ +export function runAtTargetFps(fps: number, func: () => T): T | undefined { + 'worklet' + // @ts-expect-error + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const funcId = func.__workletHash ?? '1' + + const targetIntervalMs = 1000 / fps // <-- 60 FPS => 16,6667ms interval + const now = performance.now() + const diffToLastCall = now - getLastFrameProcessorCall(funcId) + if (diffToLastCall >= targetIntervalMs) { + setLastFrameProcessorCall(funcId, now) + // Last Frame Processor call is already so long ago that we want to make a new call + return func() + } + return undefined +} diff --git a/package/src/frame-processors/throwErrorOnJS.ts b/package/src/frame-processors/throwErrorOnJS.ts new file mode 100644 index 0000000000..ce692c3fa1 --- /dev/null +++ b/package/src/frame-processors/throwErrorOnJS.ts @@ -0,0 +1,51 @@ +import { WorkletsProxy } from '../dependencies/WorkletsProxy' +import { FrameProcessorsUnavailableError } from './FrameProcessorsUnavailableError' + +interface IErrorUtils { + reportFatalError: (error: unknown) => void +} + +/** + * Rethrows the given message and stack as a JS Error on the JS Thread. + */ +let rethrowErrorOnJS: (message: string, stack: string | undefined) => void + +try { + const Worklets = WorkletsProxy.Worklets + rethrowErrorOnJS = Worklets.createRunOnJS((message: string, stack: string | undefined) => { + const error = new Error() + error.message = message + error.stack = stack + error.name = 'Frame Processor Error' + // @ts-expect-error this is react-native specific + error.jsEngine = 'VisionCamera' + + // From react-native: + // @ts-expect-error it's untyped + const errorUtils = (global.ErrorUtils ?? global.__ErrorUtils) as IErrorUtils | undefined + if (errorUtils != null && typeof errorUtils.reportFatalError === 'function') { + // we can use the JS error reporter view from react native + errorUtils.reportFatalError(error) + } else { + // just log it to console.error as a fallback + console.error('Frame Processor Error:', error) + } + }) +} catch (e) { + // react-native-worklets-core is not installed! + // Just use dummy implementations that will throw when the user tries to use Frame Processors. + rethrowErrorOnJS = () => { + throw new FrameProcessorsUnavailableError(e) + } +} + +/** + * Throws the given Error on the JS Thread using React Native's error reporter. + * @param error An {@linkcode Error}, or an object with a `message` property, otherwise a default messageg will be thrown. + */ +export function throwErrorOnJS(error: unknown): void { + 'worklet' + const safeError = error as Error | undefined + const message = safeError != null && 'message' in safeError ? safeError.message : 'Frame Processor threw an error.' + rethrowErrorOnJS(message, safeError?.stack) +} diff --git a/package/src/frame-processors/withFrameRefCounting.ts b/package/src/frame-processors/withFrameRefCounting.ts new file mode 100644 index 0000000000..29bc005da0 --- /dev/null +++ b/package/src/frame-processors/withFrameRefCounting.ts @@ -0,0 +1,26 @@ +import type { Frame, FrameInternal } from '../types/Frame' +import { throwErrorOnJS } from './throwErrorOnJS' + +/** + * A private API to wrap a Frame Processor with a ref-counting mechanism + * @worklet + * @internal + */ +export function withFrameRefCounting(frameProcessor: (frame: Frame) => void): (frame: Frame) => void { + return (frame) => { + 'worklet' + // Increment ref-count by one + const internal = frame as FrameInternal + internal.incrementRefCount() + try { + // Call sync frame processor + frameProcessor(frame) + } catch (e) { + // Re-throw error on JS Thread + throwErrorOnJS(e) + } finally { + // Potentially delete Frame if we were the last ref (no runAsync) + internal.decrementRefCount() + } + } +} diff --git a/package/src/hooks/useFrameProcessor.ts b/package/src/hooks/useFrameProcessor.ts index 8afa677a59..a7348927be 100644 --- a/package/src/hooks/useFrameProcessor.ts +++ b/package/src/hooks/useFrameProcessor.ts @@ -1,6 +1,6 @@ import type { DependencyList } from 'react' import { useMemo } from 'react' -import { wrapFrameProcessorWithRefCounting } from '../FrameProcessorPlugins' +import { withFrameRefCounting } from '../frame-processors/withFrameRefCounting' import type { ReadonlyFrameProcessor } from '../types/CameraProps' import type { Frame } from '../types/Frame' @@ -15,7 +15,7 @@ import type { Frame } from '../types/Frame' */ export function createFrameProcessor(frameProcessor: (frame: Frame) => void): ReadonlyFrameProcessor { return { - frameProcessor: wrapFrameProcessorWithRefCounting(frameProcessor), + frameProcessor: withFrameRefCounting(frameProcessor), type: 'readonly', } } diff --git a/package/src/index.ts b/package/src/index.ts index 79a4adbb90..0854255e25 100644 --- a/package/src/index.ts +++ b/package/src/index.ts @@ -1,7 +1,8 @@ +// Base Camera Exports export * from './Camera' export * from './CameraError' -export * from './FrameProcessorPlugins' +// Types export * from './types/CameraDevice' export * from './types/CameraProps' export * from './types/Frame' @@ -13,10 +14,12 @@ export * from './types/Point' export * from './types/VideoFile' export * from './types/CodeScanner' +// Devices API export * from './devices/getCameraFormat' export * from './devices/getCameraDevice' export * from './devices/Templates' +// Hooks export * from './hooks/useCameraDevice' export * from './hooks/useCameraDevices' export * from './hooks/useCameraFormat' @@ -24,4 +27,12 @@ export * from './hooks/useCameraPermission' export * from './hooks/useCodeScanner' export * from './hooks/useFrameProcessor' +// Frame Processors +export * from './frame-processors/initFrameProcessorPlugin' +export * from './frame-processors/runAsync' +export * from './frame-processors/runAtTargetFps' +// DEPRECATED: This will be removed in favour of a CxxTurboModule in the future. +export * from './frame-processors/VisionCameraProxy' + +// Skia Frame Processors export * from './skia/useSkiaFrameProcessor' diff --git a/package/src/skia/useSkiaFrameProcessor.ts b/package/src/skia/useSkiaFrameProcessor.ts index 3feb33630d..d0363f0602 100644 --- a/package/src/skia/useSkiaFrameProcessor.ts +++ b/package/src/skia/useSkiaFrameProcessor.ts @@ -2,12 +2,13 @@ import type { Frame, FrameInternal } from '../types/Frame' import type { DependencyList } from 'react' import { useEffect, useMemo } from 'react' import type { Orientation } from '../types/Orientation' -import { VisionCameraProxy, wrapFrameProcessorWithRefCounting } from '../FrameProcessorPlugins' import type { DrawableFrameProcessor } from '../types/CameraProps' import type { ISharedValue, IWorkletNativeApi } from 'react-native-worklets-core' import { WorkletsProxy } from '../dependencies/WorkletsProxy' import type { SkCanvas, SkPaint, SkImage, SkSurface } from '@shopify/react-native-skia' import { SkiaProxy } from '../dependencies/SkiaProxy' +import { withFrameRefCounting } from '../frame-processors/withFrameRefCounting' +import { VisionCameraProxy } from '../frame-processors/VisionCameraProxy' /** * Represents a Camera Frame that can be directly drawn to using Skia. @@ -197,7 +198,7 @@ export function createSkiaFrameProcessor( } return { - frameProcessor: wrapFrameProcessorWithRefCounting((frame) => { + frameProcessor: withFrameRefCounting((frame) => { 'worklet' // 1. Set up Skia Surface with size of Frame