diff --git a/packages/sdk/tests-qvac/package.json b/packages/sdk/tests-qvac/package.json index 65d53c2d45..b947e6fb10 100644 --- a/packages/sdk/tests-qvac/package.json +++ b/packages/sdk/tests-qvac/package.json @@ -8,9 +8,10 @@ "prepare:sdk": "cd .. && (bun i && bun run build); cd tests-qvac", "build": "npm run clean && tsc", "install:build": "npm install --install-links && npm run build", - "install:build:full": "npm run prepare:sdk && npm run install:build", + "install:build:full": "npm run prepare:sdk && npm run clean:sdk-snapshot && npm run install:build", "typecheck": "tsc --noEmit", "clean": "rm -rf dist", + "clean:sdk-snapshot": "rm -rf node_modules/@qvac/sdk build/consumers/ios/node_modules/@qvac/sdk build/consumers/android/node_modules/@qvac/sdk", "consumer:desktop:run": "qvac-test run:consumer:desktop" }, "dependencies": { diff --git a/packages/sdk/tests-qvac/tests/desktop/consumer.ts b/packages/sdk/tests-qvac/tests/desktop/consumer.ts index df2124255e..0e3af4b48f 100644 --- a/packages/sdk/tests-qvac/tests/desktop/consumer.ts +++ b/packages/sdk/tests-qvac/tests/desktop/consumer.ts @@ -223,7 +223,6 @@ const referenceAudioPath = path.resolve(process.cwd(), "assets/audio/transcripti resources.define("tts-chatterbox", { constant: TTS_TOKENIZER_EN_CHATTERBOX, type: "tts", - skipPreDownload: true, preLoadUnload: true, config: { ttsEngine: "chatterbox", @@ -251,7 +250,6 @@ const ttsSupertonicBaseConfig = { resources.define("tts-supertonic", { constant: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32, type: "onnx-tts", - skipPreDownload: true, preLoadUnload: true, config: { ...ttsSupertonicBaseConfig, @@ -262,7 +260,6 @@ resources.define("tts-supertonic", { resources.define("tts-supertonic-multilingual", { constant: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32, type: "onnx-tts", - skipPreDownload: true, preLoadUnload: true, config: { ...ttsSupertonicBaseConfig, @@ -275,7 +272,6 @@ resources.define("tts-supertonic-multilingual", { resources.define("parakeet-tdt", { constant: PARAKEET_TDT_ENCODER_INT8, type: "parakeet", - skipPreDownload: true, preLoadUnload: true, config: { parakeetEncoderSrc: PARAKEET_TDT_ENCODER_INT8, @@ -289,7 +285,6 @@ resources.define("parakeet-tdt", { resources.define("parakeet-ctc", { constant: PARAKEET_CTC_FP32, type: "parakeet", - skipPreDownload: true, preLoadUnload: true, config: { modelType: "ctc", @@ -302,7 +297,6 @@ resources.define("parakeet-ctc", { resources.define("parakeet-sortformer", { constant: PARAKEET_SORTFORMER_FP32, type: "parakeet", - skipPreDownload: true, preLoadUnload: true, config: { modelType: "sortformer", @@ -313,7 +307,6 @@ resources.define("parakeet-sortformer", { resources.define("vision", { constant: SMOLVLM2_500M_MULTIMODAL_Q8_0, type: "llm", - skipPreDownload: true, preLoadUnload: true, config: { ctx_size: 1024, @@ -324,7 +317,6 @@ resources.define("vision", { resources.define("diffusion", { constant: FLUX_2_KLEIN_4B_Q4_0, type: "diffusion", - skipPreDownload: true, preLoadUnload: true, config: { device: "gpu", diff --git a/packages/sdk/tests-qvac/tests/mobile/asset-uri.ts b/packages/sdk/tests-qvac/tests/mobile/asset-uri.ts new file mode 100644 index 0000000000..5f768f2039 --- /dev/null +++ b/packages/sdk/tests-qvac/tests/mobile/asset-uri.ts @@ -0,0 +1,23 @@ +/** + * Resolve a bundled RN asset module to a real filesystem path. + * + * On Android release builds expo-asset pre-sets `downloaded=true` and + * `localUri` to a drawable resource name (e.g. `assets_images_cat`), + * bypassing the native downloadAsync() that copies the resource to a + * cache file. Resetting `downloaded=false` forces the native module to + * run and produce a real `file://` URI — which we strip + percent-decode + * because SDK native addons (whisper, FFmpeg, ONNX runtime) expect POSIX + * paths, not URIs. + */ +export async function resolveBundledAssetUri(assetModule: number): Promise { + // @ts-ignore - expo-asset is a peer dependency available in mobile context + const { Asset } = await import("expo-asset"); + const asset = Asset.fromModule(assetModule); + asset.downloaded = false; + await asset.downloadAsync(); + const rawUri: string | undefined = asset.localUri ?? asset.uri; + if (!rawUri) { + throw new Error(`Failed to resolve asset: ${asset.name ?? "unknown"}`); + } + return decodeURIComponent(rawUri.replace(/^file:\/\//, "")); +} diff --git a/packages/sdk/tests-qvac/tests/mobile/consumer.ts b/packages/sdk/tests-qvac/tests/mobile/consumer.ts index 00c1fb675f..dc51377104 100644 --- a/packages/sdk/tests-qvac/tests/mobile/consumer.ts +++ b/packages/sdk/tests-qvac/tests/mobile/consumer.ts @@ -41,6 +41,7 @@ import { SD_V2_1_1B_Q8_0, } from "@qvac/sdk"; import { ResourceManager } from "../shared/resource-manager.js"; +import { resolveBundledAssetUri } from "./asset-uri.js"; import { ModelLoadingExecutor } from "../shared/executors/model-loading-executor.js"; import { CompletionExecutor } from "../shared/executors/completion-executor.js"; import { EmbeddingExecutor } from "../shared/executors/embedding-executor.js"; @@ -215,12 +216,28 @@ resources.define("afriquegemma", { }, }); +/** Look up a bundled audio file by name and resolve it to a POSIX path. */ +async function resolveBundledAudioUri(filename: string): Promise { + // @ts-ignore - assets.ts generated at consumer build time (consumer root, 3 levels up from dist/tests/mobile/) + const assets = await import("../../../assets"); + const assetModule = assets.audio?.[filename]; + if (!assetModule) { + console.warn(`[tts-chatterbox] reference audio not in registry: ${filename}`); + return undefined; + } + try { + return await resolveBundledAssetUri(assetModule); + } catch (err) { + console.warn(`[tts-chatterbox] failed to resolve ${filename}:`, err); + return undefined; + } +} resources.define("tts-chatterbox", { constant: TTS_TOKENIZER_EN_CHATTERBOX, type: "tts", - skipPreDownload: true, - config: { + preLoadUnload: true, + config: async () => ({ ttsEngine: "chatterbox", language: "en", ttsTokenizerSrc: TTS_TOKENIZER_EN_CHATTERBOX, @@ -228,7 +245,8 @@ resources.define("tts-chatterbox", { ttsEmbedTokensSrc: TTS_EMBED_TOKENS_EN_CHATTERBOX_FP32, ttsConditionalDecoderSrc: TTS_CONDITIONAL_DECODER_EN_CHATTERBOX_FP32, ttsLanguageModelSrc: TTS_LANGUAGE_MODEL_EN_CHATTERBOX_FP32, - }, + referenceAudioSrc: await resolveBundledAudioUri("transcription-short-wav.wav"), + }), }); const ttsSupertonicBaseConfig = { @@ -245,7 +263,7 @@ const ttsSupertonicBaseConfig = { resources.define("tts-supertonic", { constant: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32, type: "onnx-tts", - skipPreDownload: true, + preLoadUnload: true, config: { ...ttsSupertonicBaseConfig, language: "en", @@ -255,7 +273,7 @@ resources.define("tts-supertonic", { resources.define("tts-supertonic-multilingual", { constant: TTS_SUPERTONIC2_OFFICIAL_TEXT_ENCODER_SUPERTONE_FP32, type: "onnx-tts", - skipPreDownload: true, + preLoadUnload: true, config: { ...ttsSupertonicBaseConfig, language: "es", @@ -267,7 +285,7 @@ resources.define("tts-supertonic-multilingual", { resources.define("parakeet-tdt", { constant: PARAKEET_TDT_ENCODER_INT8, type: "parakeet", - skipPreDownload: true, + preLoadUnload: true, config: { parakeetEncoderSrc: PARAKEET_TDT_ENCODER_INT8, parakeetDecoderSrc: PARAKEET_TDT_DECODER_INT8, @@ -280,7 +298,7 @@ resources.define("parakeet-tdt", { resources.define("parakeet-ctc", { constant: PARAKEET_CTC_FP32, type: "parakeet", - skipPreDownload: true, + preLoadUnload: true, config: { modelType: "ctc", parakeetCtcModelSrc: PARAKEET_CTC_FP32, @@ -292,7 +310,7 @@ resources.define("parakeet-ctc", { resources.define("parakeet-sortformer", { constant: PARAKEET_SORTFORMER_FP32, type: "parakeet", - skipPreDownload: true, + preLoadUnload: true, config: { modelType: "sortformer", parakeetSortformerSrc: PARAKEET_SORTFORMER_FP32, @@ -302,7 +320,7 @@ resources.define("parakeet-sortformer", { resources.define("vision", { constant: SMOLVLM2_500M_MULTIMODAL_Q8_0, type: "llm", - skipPreDownload: true, + preLoadUnload: true, config: { ctx_size: 1024, projectionModelSrc: MMPROJ_SMOLVLM2_500M_MULTIMODAL_Q8_0, @@ -370,6 +388,10 @@ export const executor = createExecutor({ new SkipExecutor(/^translation-afriquegemma-/, "AfriqueGemma 4B (~2.7 GB) exceeds iOS memory budget"), // TODO(QVAC-18460): re-enable once iOS transcribe() crash is fixed. new SkipExecutor(/^transcription-/, "TODO(QVAC-18460): transcription disabled on iOS — transcribe() hard-crashes consumer after FFmpegDecoder unload"), + skipTests([ + "config-reload-then-transcribe", + "error-transcription-failed", + ], "TODO(QVAC-18460): transcribe() hard-crashes consumer on iOS"), ] : []), // Real executors diff --git a/packages/sdk/tests-qvac/tests/mobile/executors/model-asset-executor.ts b/packages/sdk/tests-qvac/tests/mobile/executors/model-asset-executor.ts index 10f3598938..fce216c176 100644 --- a/packages/sdk/tests-qvac/tests/mobile/executors/model-asset-executor.ts +++ b/packages/sdk/tests-qvac/tests/mobile/executors/model-asset-executor.ts @@ -1,6 +1,7 @@ import { AssetExecutor, type TestDefinitions } from "@tetherto/qvac-test-suite/mobile"; import type { ResourceManager } from "../../shared/resource-manager.js"; import { modelSetup, modelTeardown } from "../../shared/resource-lifecycle.js"; +import { resolveBundledAssetUri } from "../asset-uri.js"; /** * Combines AssetExecutor's resolveAsset() with the shared model @@ -13,28 +14,8 @@ export abstract class ModelAssetExecutor< super(); } - /** - * On Android release builds, images are bundled as drawable resources. - * expo-asset pre-sets downloaded=true and localUri to the drawable resource - * name (e.g. "assets_images_cat"), bypassing the native downloadAsync() - * that would copy the resource to a real cache file. Resetting downloaded=false - * forces the native module to run, which uses Android's ResourceManager to - * copy the drawable to the ExponentAsset cache and returns a real file:// URI. - */ protected override async resolveAsset(assetModule: number): Promise { - // @ts-ignore - expo-asset is a peer dependency available in mobile context - const { Asset } = await import("expo-asset"); - const asset = Asset.fromModule(assetModule); - asset.downloaded = false; - await asset.downloadAsync(); - let uri: string = asset.localUri || asset.uri; - if (!uri) { - throw new Error(`Failed to resolve asset: ${asset.name ?? "unknown"}`); - } - if (uri.startsWith("file://")) { - uri = uri.substring(7); - } - return decodeURIComponent(uri); + return await resolveBundledAssetUri(assetModule); } async setup(testId: string, context: unknown) { diff --git a/packages/sdk/tests-qvac/tests/mobile/executors/tts-executor.ts b/packages/sdk/tests-qvac/tests/mobile/executors/tts-executor.ts index 5a4bfc02a8..e670ff77c0 100644 --- a/packages/sdk/tests-qvac/tests/mobile/executors/tts-executor.ts +++ b/packages/sdk/tests-qvac/tests/mobile/executors/tts-executor.ts @@ -17,7 +17,7 @@ export class MobileTtsExecutor extends ModelAssetExecutor { protected handlers = Object.fromEntries( ttsTests.map((test) => { const params = test.params as TtsParams; - const dep = test.testId.startsWith("tts-supertonic-") ? "tts-supertonic" : "tts-chatterbox"; + const dep = test.metadata?.dependency || "tts-chatterbox"; if (params.stream && params.sentenceStream) { return [test.testId, this.makeSentenceStream(dep)]; } @@ -30,51 +30,10 @@ export class MobileTtsExecutor extends ModelAssetExecutor { ) as never; protected defaultHandler = undefined; - private audioAssets: Record | null = null; - private referenceAudioPatched = false; - constructor(resources: ResourceManager) { super(resources); } - async setup(testId: string, context: unknown) { - if (!this.referenceAudioPatched) { - await this.patchChatterboxReferenceAudio(); - this.referenceAudioPatched = true; - } - await super.setup(testId, context); - } - - private async loadAudioAssets() { - if (!this.audioAssets) { - // @ts-ignore - assets.ts is generated at consumer build time - const assets = await import("../../../../assets"); - this.audioAssets = assets.audio; - } - return this.audioAssets!; - } - - /** - * Resolve the reference audio asset URI and patch the tts-chatterbox - * resource definition config. Must run before the first ensureLoaded() - * call so loadModel() receives the referenceAudioSrc. - */ - private async patchChatterboxReferenceAudio() { - try { - const audio = await this.loadAudioAssets(); - const assetModule = audio["transcription-short-wav.wav"]; - if (!assetModule) return; - - const audioUri = await this.resolveAsset(assetModule); - const def = (this.resources as unknown as { definitions: Map }> }).definitions.get("tts-chatterbox"); - if (def?.config) { - def.config.referenceAudioSrc = audioUri; - } - } catch (e) { - console.warn("Failed to resolve chatterbox reference audio:", e); - } - } - private makeNonStreaming(dep: string, isEmptyTest: boolean) { return async (params: TtsParams, expectation: Expectation): Promise => { const modelId = await this.resources.ensureLoaded(dep); diff --git a/packages/sdk/tests-qvac/tests/shared/resource-manager.ts b/packages/sdk/tests-qvac/tests/shared/resource-manager.ts index a0382f28a4..34ce38f9b9 100644 --- a/packages/sdk/tests-qvac/tests/shared/resource-manager.ts +++ b/packages/sdk/tests-qvac/tests/shared/resource-manager.ts @@ -1,10 +1,14 @@ import { loadModel, downloadAsset, unloadModel, cancel } from "@qvac/sdk"; import type { ModelConstant } from "@qvac/sdk"; +type ModelConfig = Record; +type ModelConfigResolver = () => Promise; + interface ModelDefinition { constant: ModelConstant; type: string; - config?: Record; + /** Static config or async resolver (cached per-dep) for runtime-only fields like RN asset URIs. */ + config?: ModelConfig | ModelConfigResolver; skipPreDownload?: boolean; preLoadUnload?: true; } @@ -37,6 +41,7 @@ export interface ResourceManagerOptions { export class ResourceManager { private definitions = new Map(); + private resolvedConfigs = new Map(); private models = new Map(); private testCount = 0; private downloaded = false; @@ -46,6 +51,15 @@ export class ResourceManager { this.unloadSettleMs = options.unloadSettleMs ?? 0; } + private async resolveConfig(dep: string, def: ModelDefinition): Promise { + if (typeof def.config !== "function") return def.config; + const cached = this.resolvedConfigs.get(dep); + if (cached) return cached; + const resolved = await def.config(); + this.resolvedConfigs.set(dep, resolved); + return resolved; + } + define(dep: string, definition: ModelDefinition) { this.definitions.set(dep, definition); } @@ -106,7 +120,7 @@ export class ResourceManager { const modelId = await loadModel({ modelSrc: def.constant as never, modelType: def.type, - modelConfig: def.config, + modelConfig: await this.resolveConfig(dep, def), }); log?.(`✅ pre-loaded ${dep}: ${def.constant.name} - unloading...`); await unloadModel({ modelId }); @@ -140,7 +154,7 @@ export class ResourceManager { const modelId = await loadModel({ modelSrc: def.constant as never, modelType: def.type as "llm" | "whisper" | "embeddings", - modelConfig: def.config, + modelConfig: await this.resolveConfig(dep, def), }); this.models.set(dep, {