Support React Native

package.json

@@ -62,10 +62,18 @@
   "homepage": "https://github.com/huggingface/transformers.js#readme",
   "dependencies": {
     "@huggingface/jinja": "^0.3.0",
+    "image-codecs": "^0.1.0",
     "onnxruntime-node": "1.19.2",
     "onnxruntime-web": "1.20.0-dev.20241016-2b8fc5529b",
+    "onnxruntime-react-native": "1.19.2",
     "sharp": "^0.33.5"
   },
+  "optionalDependencies": {
+    "path-browserify": "^1.0.1"
+  },
+  "peerDependencies": {
+    "react-native-fs": "*"
+  },
   "devDependencies": {
     "@types/jest": "^29.5.1",
     "@webgpu/types": "^0.1.44",
@@ -87,8 +95,18 @@
     "README.md",
     "LICENSE"
   ],
+  "react-native": {
+    "fs": "react-native-fs",
+    "onnxruntime-web": false,
+    "onnxruntime-node": "onnxruntime-react-native",
+    "image-codecs": "image-codecs",
+    "sharp": false,
+    "path": "path-browserify",
+    "stream/web": false
+  },
   "browser": {
     "fs": false,
+    "image-codecs": false,
     "path": false,
     "url": false,
     "sharp": false,

src/backends/onnx.js

@@ -65,6 +65,12 @@ if (ORT_SYMBOL in globalThis) {
   // If the JS runtime exposes their own ONNX runtime, use it
   ONNX = globalThis[ORT_SYMBOL];
 
+} else if (apis.IS_REACT_NATIVE_ENV) {
+    ONNX = ONNX_NODE.default ?? ONNX_NODE;
+
+    supportedDevices.push('xnnpack', 'cpu');
+    defaultDevices = ['xnnpack', 'cpu'];
+
 } else if (apis.IS_NODE_ENV) {
     ONNX = ONNX_NODE.default ?? ONNX_NODE;
 
@@ -146,19 +152,19 @@ let wasmInitPromise = null;
 
 /**
  * Create an ONNX inference session.
- * @param {Uint8Array} buffer The ONNX model buffer.
+ * @param {Uint8Array|string} bufferOrPath The ONNX model buffer.
  * @param {import('onnxruntime-common').InferenceSession.SessionOptions} session_options ONNX inference session options.
  * @param {Object} session_config ONNX inference session configuration.
  * @returns {Promise<import('onnxruntime-common').InferenceSession & { config: Object}>} The ONNX inference session.
  */
-export async function createInferenceSession(buffer, session_options, session_config) {
+export async function createInferenceSession(bufferOrPath, session_options, session_config) {
     if (wasmInitPromise) {
         // A previous session has already initialized the WASM runtime
         // so we wait for it to resolve before creating this new session.
         await wasmInitPromise;
     }
 
-    const sessionPromise = InferenceSession.create(buffer, session_options);
+    const sessionPromise = InferenceSession.create(bufferOrPath, session_options);
     wasmInitPromise ??= sessionPromise;
     const session = await sessionPromise;
     session.config = session_config;

src/env.js

@@ -40,6 +40,8 @@ const IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === 'node';
 const IS_FS_AVAILABLE = !isEmpty(fs);
 const IS_PATH_AVAILABLE = !isEmpty(path);
 
+const IS_REACT_NATIVE_ENV = typeof navigator !== 'undefined' && navigator.product === 'ReactNative';
+
 /**
  * A read-only object containing information about the APIs available in the current environment.
  */
@@ -50,6 +52,9 @@ export const apis = Object.freeze({
     /** Whether we are running in a web worker environment */
     IS_WEBWORKER_ENV,
 
+    /** Whether we are running in a React Native environment */
+    IS_REACT_NATIVE_ENV,
+
     /** Whether the Cache API is available */
     IS_WEB_CACHE_AVAILABLE,
 
@@ -73,9 +78,12 @@ export const apis = Object.freeze({
 });
 
 const RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE;
-const dirname__ = RUNNING_LOCALLY
-    ? path.dirname(path.dirname(url.fileURLToPath(import.meta.url)))
-    : './';
+let dirname__ = './';
+if (IS_REACT_NATIVE_ENV) {
+    dirname__ = fs.DocumentDirectoryPath;
+} else if (RUNNING_LOCALLY) {
+    dirname__ = path.dirname(path.dirname(url.fileURLToPath(import.meta.url)));
+}
 
 // Only used for environments with access to file system
 const DEFAULT_CACHE_DIR = RUNNING_LOCALLY
@@ -117,7 +125,7 @@ export const env = {
     /////////////////// Backends settings ///////////////////
     // NOTE: These will be populated later by the backends themselves.
     backends: {
-        // onnxruntime-web/onnxruntime-node
+        // onnxruntime-web/onnxruntime-node/onnxruntime-react-native
         onnx: {},
     },
 
@@ -130,6 +138,8 @@ export const env = {
     localModelPath: localModelPath,
     useFS: IS_FS_AVAILABLE,
 
+    rnUseCanvas: true,
+
     /////////////////// Cache settings ///////////////////
     useBrowserCache: IS_WEB_CACHE_AVAILABLE,
 

src/models.js

@@ -68,6 +68,7 @@ import {
 
 import {
     getModelFile,
+    getModelPath,
     getModelJSON,
 } from './utils/hub.js';
 
@@ -117,6 +118,8 @@ import { apis } from './env.js';
 import { WhisperGenerationConfig } from './models/whisper/generation_whisper.js';
 import { whisper_language_to_code } from './models/whisper/common_whisper.js';
 
+const IS_BROWSER = typeof navigator !== 'undefined' && navigator.product !== 'ReactNative';
+
 //////////////////////////////////////////////////
 // Model types: used internally
 const MODEL_TYPES = {
@@ -146,7 +149,7 @@ const MODEL_CLASS_TO_NAME_MAPPING = new Map();
  * @param {string} pretrained_model_name_or_path The path to the directory containing the model file.
  * @param {string} fileName The name of the model file.
  * @param {import('./utils/hub.js').PretrainedModelOptions} options Additional options for loading the model.
- * @returns {Promise<{buffer: Uint8Array, session_options: Object, session_config: Object}>} A Promise that resolves to the data needed to create an InferenceSession object.
+ * @returns {Promise<{bufferOrPath: Uint8Array|string, session_options: Object, session_config: Object}>} A Promise that resolves to the data needed to create an InferenceSession object.
  * @private
  */
 async function getSession(pretrained_model_name_or_path, fileName, options) {
@@ -223,63 +226,103 @@ async function getSession(pretrained_model_name_or_path, fileName, options) {
         );
     }
 
-    const bufferPromise = getModelFile(pretrained_model_name_or_path, modelFileName, true, options);
+    let bufferOrPath;
 
-    // handle onnx external data files
-    const use_external_data_format = options.use_external_data_format ?? custom_config.use_external_data_format;
-    /** @type {Promise<{path: string, data: Uint8Array}>[]} */
-    let externalDataPromises = [];
-    if (use_external_data_format && (
-        use_external_data_format === true ||
-        (
-            typeof use_external_data_format === 'object' &&
-            use_external_data_format.hasOwnProperty(fileName) &&
-            use_external_data_format[fileName] === true
-        )
-    )) {
-        if (apis.IS_NODE_ENV) {
-            throw new Error('External data format is not yet supported in Node.js');
+    if (apis.IS_NODE_ENV || apis.IS_REACT_NATIVE_ENV) {
+
+        const pathPromise = getModelPath(pretrained_model_name_or_path, modelFileName, true, options);
+        // handle onnx external data files
+        const use_external_data_format = options.use_external_data_format ?? custom_config.use_external_data_format;
+        /** @type {Promise<{path: string, data: string}>[]} */
+        let externalDataPromises = [];
+        if (use_external_data_format && (
+            use_external_data_format === true ||
+            (
+                typeof use_external_data_format === 'object' &&
+                use_external_data_format.hasOwnProperty(fileName) &&
+                use_external_data_format[fileName] === true
+            )
+        )) {
+            const path = `${fileName}${suffix}.onnx_data`;
+            const fullPath = `${options.subfolder ?? ''}/${path}`;
+            externalDataPromises.push(new Promise(async (resolve, reject) => {
+                const data = await getModelPath(pretrained_model_name_or_path, fullPath, true, options);
+                resolve({ path, data })
+            }));
+
+        } else if (session_options.externalData !== undefined) {
+            externalDataPromises = session_options.externalData.map(async (ext) => {
+                // if the external data is a string, fetch the file and replace the string with its content
+                if (typeof ext.data === "string") {
+                    const ext_buffer = await getModelPath(pretrained_model_name_or_path, ext.data, true, options);
+                    return { ...ext, data: ext_buffer };
+                }
+                return ext;
+            });
         }
-        const path = `${fileName}${suffix}.onnx_data`;
-        const fullPath = `${options.subfolder ?? ''}/${path}`;
-        externalDataPromises.push(new Promise(async (resolve, reject) => {
-            const data = await getModelFile(pretrained_model_name_or_path, fullPath, true, options);
-            resolve({ path, data })
-        }));
-
-    } else if (session_options.externalData !== undefined) {
-        externalDataPromises = session_options.externalData.map(async (ext) => {
-            // if the external data is a string, fetch the file and replace the string with its content
-            if (typeof ext.data === "string") {
-                const ext_buffer = await getModelFile(pretrained_model_name_or_path, ext.data, true, options);
-                return { ...ext, data: ext_buffer };
-            }
-            return ext;
-        });
-    }
 
-    if (externalDataPromises.length > 0) {
-        session_options.externalData = await Promise.all(externalDataPromises);
-    }
+        if (externalDataPromises.length > 0) {
+            session_options.externalData = await Promise.all(externalDataPromises);
+        }
 
-    if (selectedDevice === 'webgpu') {
-        const shapes = getKeyValueShapes(options.config, {
-            prefix: 'present',
-        });
-        if (Object.keys(shapes).length > 0 && !isONNXProxy()) {
-            // Only set preferredOutputLocation if shapes are present and we aren't proxying ONNX
-            /** @type {Record<string, import('onnxruntime-common').Tensor.DataLocation>} */
-            const preferredOutputLocation = {};
-            for (const key in shapes) {
-                preferredOutputLocation[key] = 'gpu-buffer';
+        bufferOrPath = await pathPromise;
+    } else {
+
+        const bufferPromise = getModelFile(pretrained_model_name_or_path, modelFileName, true, options);
+
+        // handle onnx external data files
+        const use_external_data_format = options.use_external_data_format ?? custom_config.use_external_data_format;
+        /** @type {Promise<{path: string, data: Uint8Array}>[]} */
+        let externalDataPromises = [];
+        if (use_external_data_format && (
+            use_external_data_format === true ||
+            (
+                typeof use_external_data_format === 'object' &&
+                use_external_data_format.hasOwnProperty(fileName) &&
+                use_external_data_format[fileName] === true
+            )
+        )) {
+            const path = `${fileName}${suffix}.onnx_data`;
+            const fullPath = `${options.subfolder ?? ''}/${path}`;
+            externalDataPromises.push(new Promise(async (resolve, reject) => {
+                const data = await getModelFile(pretrained_model_name_or_path, fullPath, true, options);
+                resolve({ path, data })
+            }));
+
+        } else if (session_options.externalData !== undefined) {
+            externalDataPromises = session_options.externalData.map(async (ext) => {
+                // if the external data is a string, fetch the file and replace the string with its content
+                if (typeof ext.data === "string") {
+                    const ext_buffer = await getModelFile(pretrained_model_name_or_path, ext.data, true, options);
+                    return { ...ext, data: ext_buffer };
+                }
+                return ext;
+            });
+        }
+
+        if (externalDataPromises.length > 0) {
+            session_options.externalData = await Promise.all(externalDataPromises);
+        }
+
+        if (selectedDevice === 'webgpu') {
+            const shapes = getKeyValueShapes(options.config, {
+                prefix: 'present',
+            });
+            if (Object.keys(shapes).length > 0 && !isONNXProxy()) {
+                // Only set preferredOutputLocation if shapes are present and we aren't proxying ONNX
+                /** @type {Record<string, import('onnxruntime-common').Tensor.DataLocation>} */
+                const preferredOutputLocation = {};
+                for (const key in shapes) {
+                    preferredOutputLocation[key] = 'gpu-buffer';
+                }
+                session_options.preferredOutputLocation = preferredOutputLocation;
             }
-            session_options.preferredOutputLocation = preferredOutputLocation;
         }
-    }
 
-    const buffer = await bufferPromise;
+        bufferOrPath = await bufferPromise;
+    }
 
-    return { buffer, session_options, session_config };
+    return { bufferOrPath, session_options, session_config };
 }
 
 /**
@@ -294,8 +337,8 @@ async function getSession(pretrained_model_name_or_path, fileName, options) {
 async function constructSessions(pretrained_model_name_or_path, names, options) {
     return Object.fromEntries(await Promise.all(
         Object.keys(names).map(async (name) => {
-            const { buffer, session_options, session_config } = await getSession(pretrained_model_name_or_path, names[name], options);
-            const session = await createInferenceSession(buffer, session_options, session_config);
+            const { bufferOrPath, session_options, session_config } = await getSession(pretrained_model_name_or_path, names[name], options);
+            const session = await createInferenceSession(bufferOrPath, session_options, session_config);
             return [name, session];
         })
     ));
@@ -342,7 +385,7 @@ function validateInputs(session, inputs) {
             missingInputs.push(inputName);
             continue;
         }
-        // NOTE: When `env.wasm.proxy is true` the tensor is moved across the Worker
+        // NOTE: When `onnx_env.wasm.proxy is true` the tensor is moved across the Worker
         // boundary, transferring ownership to the worker and invalidating the tensor.
         // So, in this case, we simply sacrifice a clone for it.
         checkedInputs[inputName] = isONNXProxy() ? tensor.clone() : tensor;
@@ -386,7 +429,14 @@ async function sessionRun(session, inputs) {
     } catch (e) {
         // This usually occurs when the inputs are of the wrong type.
         console.error(`An error occurred during model execution: "${e}".`);
-        console.error('Inputs given to model:', checkedInputs);
+        // Not log full data, it may cause crash in React Native
+        console.error(
+            'Inputs given to model:',
+            IS_BROWSER ? checkedInputs : Object.fromEntries(
+                Object.entries(checkedInputs)
+                    .map(([key, tensor]) => [key, { dims: tensor.dims, type: tensor.type }])
+            )
+        );
         throw e;
     }
 }

src/pipelines.js

@@ -73,6 +73,9 @@ import {
     topk,
 } from './utils/tensor.js';
 import { RawImage } from './utils/image.js';
+import {
+    fetchBinary
+} from './utils/hub.js';
 
 
 /**
@@ -2719,7 +2722,7 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi
         if (typeof speaker_embeddings === 'string' || speaker_embeddings instanceof URL) {
             // Load from URL with fetch
             speaker_embeddings = new Float32Array(
-                await (await fetch(speaker_embeddings)).arrayBuffer()
+                await (await fetchBinary(speaker_embeddings)).arrayBuffer()
             );
         }
 
@@ -3339,4 +3342,4 @@ async function loadItems(mapping, model, pretrainedOptions) {
     }
 
     return result;
-}
+}

src/tokenizers.js

@@ -38,6 +38,8 @@ import {
 import { max, min, round } from './utils/maths.js';
 import { Tensor } from './utils/tensor.js';
 
+import { env } from './env.js';
+
 import {
     PriorityQueue,
     TokenLattice,
@@ -2084,7 +2086,7 @@ class ByteLevelDecoder extends Decoder {
      */
     convert_tokens_to_string(tokens) {
         const text = tokens.join('');
-        const byteArray = new Uint8Array([...text].map(c => this.byte_decoder[c]));
+        const byteArray = Uint8Array.from([...text].map(c => this.byte_decoder[c]));
         const decoded_text = this.text_decoder.decode(byteArray);
         return decoded_text;
     }