diff --git a/packages/sdk/e2e/assets/images/diffusion-img2img-source-256.png b/packages/sdk/e2e/assets/images/diffusion-img2img-source-256.png new file mode 100644 index 0000000000..45a0c7698e Binary files /dev/null and b/packages/sdk/e2e/assets/images/diffusion-img2img-source-256.png differ diff --git a/packages/sdk/e2e/tests/desktop/executors/diffusion-executor.ts b/packages/sdk/e2e/tests/desktop/executors/diffusion-executor.ts index c773d40644..41ae9c3db7 100644 --- a/packages/sdk/e2e/tests/desktop/executors/diffusion-executor.ts +++ b/packages/sdk/e2e/tests/desktop/executors/diffusion-executor.ts @@ -1,6 +1,9 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import { DiffusionExecutor as SharedDiffusionExecutor } from "../../shared/executors/diffusion-executor.js"; +import { + DiffusionExecutor as SharedDiffusionExecutor, + type DiffusionParams, +} from "../../shared/executors/diffusion-executor.js"; function readImageBytes(name: string): Uint8Array { const fileName = name.split("/").pop()!; @@ -9,39 +12,27 @@ function readImageBytes(name: string): Uint8Array { } export class DesktopDiffusionExecutor extends SharedDiffusionExecutor { + // Resolve string filenames declared in test params to bytes via Node fs. protected override async resolveParams( - p: Record, - ): Promise> { - const out: Record = { ...p }; + p: DiffusionParams, + ): Promise { + const out: DiffusionParams = { ...p }; - if (p.init_image !== undefined) { - if (typeof p.init_image !== "string") { - throw new Error( - `init_image in test params must be a string filename, got: ${typeof p.init_image}`, - ); - } + if (typeof p.init_image === "string") { out.init_image = readImageBytes(p.init_image); } - if (p.image !== undefined) { - if (typeof p.image !== "string") { - throw new Error( - `image in test params must be a string filename, got: ${typeof p.image}`, - ); - } + if (typeof p.image === "string") { out.image = readImageBytes(p.image); } - if (p.init_images !== undefined) { - if ( - !Array.isArray(p.init_images) || - !p.init_images.every((v) => typeof v === "string") - ) { + if (Array.isArray(p.init_images)) { + if (!p.init_images.every((v): v is string => typeof v === "string")) { throw new Error( "init_images in test params must be a string[] of image filenames", ); } - out.init_images = (p.init_images as string[]).map(readImageBytes); + out.init_images = p.init_images.map(readImageBytes); } return out; diff --git a/packages/sdk/e2e/tests/diffusion-tests.ts b/packages/sdk/e2e/tests/diffusion-tests.ts index c560f00065..189fa4607f 100644 --- a/packages/sdk/e2e/tests/diffusion-tests.ts +++ b/packages/sdk/e2e/tests/diffusion-tests.ts @@ -1,21 +1,34 @@ // Diffusion test definitions import type { TestDefinition, TestResult } from "@tetherto/qvac-test-suite"; +type ExpectationLike = + | { validation: "type"; expectedType: "string" | "number" | "array" } + | { validation: "throws-error"; errorContains: string } + | { validation: "function"; fn: (result: unknown) => TestResult }; + type DiffusionTestOptions = { estimatedDurationMs?: number; suites?: string[]; dependency?: string; }; -const createDiffusionTest = ( - testId: string, - params: Record, - expectation: - | { validation: "type"; expectedType: "string" | "number" | "array" } - | { validation: "throws-error"; errorContains: string } - | { validation: "function"; fn: (result: unknown) => TestResult }, +// Generic so `typeof someTest.testId`/`typeof someTest.params` keep their literal +// types — that's what feeds `BaseExecutor`'s typed handlers map and lets each +// handler method see real `params` instead of `any`. +export type DiffusionTestDef< + TId extends string, + P extends Record, +> = TestDefinition & { testId: TId; params: P }; + +function createDiffusionTest< + const TId extends string, + const P extends Record, +>( + testId: TId, + params: P, + expectation: ExpectationLike, options: DiffusionTestOptions = {}, -): TestDefinition => { +): DiffusionTestDef { const { estimatedDurationMs = 300000, suites, @@ -31,8 +44,49 @@ const createDiffusionTest = ( dependency, estimatedDurationMs, }, + } as DiffusionTestDef; +} + +// Read PNG IHDR width/height: 8-byte signature, 4-byte chunk length, 4-byte +// "IHDR" tag, then big-endian uint32 width and uint32 height at offsets 16/20. +function readPngDims( + buf: Uint8Array, +): { width: number; height: number } | null { + if (buf.length < 24) return null; + const sig = buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4e && buf[3] === 0x47; + if (!sig) return null; + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + return { width: view.getUint32(16, false), height: view.getUint32(20, false) }; +} + +// Shared PNG-dimension validator: asserts first output is a PNG of expected size. +// `label` lets callers identify themselves in the failure message. +function validatePngDims( + expectedWidth: number, + expectedHeight: number, + label: string, +) { + return (result: unknown): TestResult => { + if (!Array.isArray(result) || result.length === 0) { + return { passed: false, output: "No outputs generated" }; + } + const out = result[0]; + if (!(out instanceof Uint8Array)) { + return { passed: false, output: "First output is not a Uint8Array" }; + } + const dims = readPngDims(out); + if (!dims) { + return { passed: false, output: "Output is not a valid PNG" }; + } + const passed = dims.width === expectedWidth && dims.height === expectedHeight; + return { + passed, + output: passed + ? `${label} OK: ${dims.width}x${dims.height}` + : `${label}: expected ${expectedWidth}x${expectedHeight}, got ${dims.width}x${dims.height}`, + }; }; -}; +} // ---- txt2img ---- @@ -154,6 +208,9 @@ export const diffusionBatchCount = createDiffusionTest( ); // ---- img2img ---- +// Source asset is 256x256 to match request width/height: FLUX.2 auto-resize is +// a no-op and SD 2.1 SDEdit emits source-sized output, so output is 256x256 on +// both engines. export const diffusionBasicImg2img = createDiffusionTest( "diffusion-basic-img2img", @@ -169,6 +226,59 @@ export const diffusionBasicImg2img = createDiffusionTest( { validation: "type", expectedType: "array" }, ); +// FLUX.2 ignores img_cfg_scale (in-context conditioning); SD 2.1 honors it via +// SDEdit. Schema accept + PNG size check is the strongest cross-platform +// assertion without per-engine branching. +export const diffusionImg2imgImgCfgScale = createDiffusionTest( + "diffusion-img2img-img-cfg-scale", + { + prompt: "oil painting style", + init_image: "diffusion-img2img-source-256.png", + strength: 0.5, + img_cfg_scale: 5.0, + width: 256, + height: 256, + steps: 4, + seed: 42, + }, + { validation: "function", fn: validatePngDims(256, 256, "img2img PNG") }, +); + +export const diffusionImg2imgVsTxt2imgBaseline = createDiffusionTest( + "diffusion-img2img-vs-txt2img-baseline", + { + prompt: "watercolor style", + init_image: "diffusion-img2img-source-256.png", + strength: 0.5, + width: 256, + height: 256, + steps: 4, + seed: 42, + }, + // Required by TestDefinition but effectively ignored — DiffusionExecutor.img2imgVsTxt2imgBaseline gates the result. + { validation: "type", expectedType: "array" }, + { estimatedDurationMs: 600000 }, +); + +export const diffusionImg2imgInvalidStrength = createDiffusionTest( + "diffusion-img2img-invalid-strength", + { + prompt: "test", + init_image: "diffusion-img2img-source-256.png", + strength: 1.5, + width: 256, + height: 256, + steps: 4, + }, + { + validation: "throws-error", + // Match the field path rather than the Zod message — stable across version + // bumps that rephrase numeric-bound messages. + errorContains: "strength", + }, + { estimatedDurationMs: 60000 }, +); + // ---- streaming ---- export const diffusionStreaming = createDiffusionTest( @@ -263,46 +373,6 @@ const ESRGAN_SOURCE_HEIGHT = 128; const STANDALONE_UPSCALER_SOURCE_WIDTH = 64; const STANDALONE_UPSCALER_SOURCE_HEIGHT = 64; -// Decode the IHDR chunk (width/height as big-endian uint32 at offsets 16 and 20) -// and assert dimensions are source * scale. -function validateEsrganUpscale(result: unknown): TestResult { - if (!Array.isArray(result) || result.length === 0) { - return { passed: false, output: "No outputs generated" }; - } - const output = result[0] as Uint8Array; - const view = new DataView(output.buffer, output.byteOffset, output.byteLength); - const width = view.getUint32(16, false); - const height = view.getUint32(20, false); - const expectedWidth = ESRGAN_SOURCE_WIDTH * ESRGAN_SCALE; - const expectedHeight = ESRGAN_SOURCE_HEIGHT * ESRGAN_SCALE; - const passed = width === expectedWidth && height === expectedHeight; - return { - passed, - output: passed - ? `ESRGAN x${ESRGAN_SCALE} upscale OK: ${ESRGAN_SOURCE_WIDTH}x${ESRGAN_SOURCE_HEIGHT} -> ${width}x${height}` - : `Expected ${expectedWidth}x${expectedHeight} from ${ESRGAN_SOURCE_WIDTH}x${ESRGAN_SOURCE_HEIGHT} input, got ${width}x${height} (upscale not applied?)`, - }; -} - -function validateStandaloneUpscale(result: unknown): TestResult { - if (!Array.isArray(result) || result.length === 0) { - return { passed: false, output: "No outputs generated" }; - } - const output = result[0] as Uint8Array; - const view = new DataView(output.buffer, output.byteOffset, output.byteLength); - const width = view.getUint32(16, false); - const height = view.getUint32(20, false); - const expectedWidth = STANDALONE_UPSCALER_SOURCE_WIDTH * ESRGAN_SCALE; - const expectedHeight = STANDALONE_UPSCALER_SOURCE_HEIGHT * ESRGAN_SCALE; - const passed = width === expectedWidth && height === expectedHeight; - return { - passed, - output: passed - ? `Standalone upscaler x${ESRGAN_SCALE} OK: ${STANDALONE_UPSCALER_SOURCE_WIDTH}x${STANDALONE_UPSCALER_SOURCE_HEIGHT} -> ${width}x${height}` - : `Expected ${expectedWidth}x${expectedHeight} from ${STANDALONE_UPSCALER_SOURCE_WIDTH}x${STANDALONE_UPSCALER_SOURCE_HEIGHT} input, got ${width}x${height}`, - }; -} - export const diffusionEsrganUpscaleX4 = createDiffusionTest( "diffusion-esrgan-upscale-x4", { @@ -313,7 +383,14 @@ export const diffusionEsrganUpscaleX4 = createDiffusionTest( seed: 42, upscale: true, }, - { validation: "function", fn: validateEsrganUpscale }, + { + validation: "function", + fn: validatePngDims( + ESRGAN_SOURCE_WIDTH * ESRGAN_SCALE, + ESRGAN_SOURCE_HEIGHT * ESRGAN_SCALE, + `ESRGAN x${ESRGAN_SCALE}`, + ), + }, { estimatedDurationMs: 600000, dependency: "diffusion-esrgan" }, ); @@ -323,7 +400,14 @@ export const diffusionStandaloneUpscalerX4 = createDiffusionTest( image: "small-64.jpg", repeats: 1, }, - { validation: "function", fn: validateStandaloneUpscale }, + { + validation: "function", + fn: validatePngDims( + STANDALONE_UPSCALER_SOURCE_WIDTH * ESRGAN_SCALE, + STANDALONE_UPSCALER_SOURCE_HEIGHT * ESRGAN_SCALE, + `Standalone upscaler x${ESRGAN_SCALE}`, + ), + }, { estimatedDurationMs: 600000, dependency: "upscaler" }, ); @@ -352,6 +436,9 @@ export const diffusionTests = [ diffusionSeedReproducibility, diffusionBatchCount, diffusionBasicImg2img, + diffusionImg2imgImgCfgScale, + diffusionImg2imgVsTxt2imgBaseline, + diffusionImg2imgInvalidStrength, diffusionStreaming, diffusionStreamingProgress, diffusionStatsPresent, @@ -361,4 +448,4 @@ export const diffusionTests = [ diffusionEsrganUpscaleX4, diffusionStandaloneUpscalerX4, diffusionEmptyPrompt, -]; +] as const; diff --git a/packages/sdk/e2e/tests/mobile/consumer.ts b/packages/sdk/e2e/tests/mobile/consumer.ts index 62fd555351..f668d21d02 100644 --- a/packages/sdk/e2e/tests/mobile/consumer.ts +++ b/packages/sdk/e2e/tests/mobile/consumer.ts @@ -28,7 +28,6 @@ import { MMPROJ_SMOLVLM2_500M_MULTIMODAL_Q8_0, SALAMANDRATA_2B_INST_Q4, AFRICAN_4B_TRANSLATION_Q4_K_M, - SD_V2_1_1B_Q8_0, } from "@qvac/sdk"; import { ResourceManager } from "../shared/resource-manager.js"; import { collectTestDeps } from "../shared/collect-test-deps.js"; @@ -58,7 +57,6 @@ import { MobileConfigReloadExecutor } from "./executors/config-reload-executor.j import { MobileTtsExecutor } from "./executors/tts-executor.js"; import { DownloadExecutor } from "../shared/executors/download-executor.js"; import { DelegatedInferenceExecutor } from "../shared/executors/delegated-inference-executor.js"; -import { MobileDiffusionExecutor } from "./executors/diffusion-executor.js"; import { LifecycleExecutor } from "../shared/executors/lifecycle-executor.js"; import { ConfigExecutor } from "../shared/executors/config-executor.js"; import { MobileCancellationExecutor } from "./executors/cancellation-executor.js"; @@ -296,17 +294,6 @@ resources.define("vision", { }, }); -resources.define("diffusion", { - constant: SD_V2_1_1B_Q8_0, - type: "diffusion", - config: { - device: "gpu", - threads: 4, - prediction: "v", - vae_on_cpu: true, - }, -}); - function skipTests(testIds: string[], reason: string) { return new SkipExecutor(new RegExp(`^(${testIds.join("|")})$`), reason); } @@ -335,7 +322,7 @@ export const executor = createExecutor({ /^video-/, "Video mode works on mobile but SDK-shipped Wan models are too large to load on-device; mobile apps should pass a `delegate` to loadModel(...), desktop covers local-load coverage", ), - new SkipExecutor(/^(diffusion-|addon-logging-diffusion$)/, "SD v2.1 1B Q8_0 cold-load is too heavy for Device Farm devices (iOS variable 5–15min, Android blocks JS thread >300s and trips heartbeat)"), + new SkipExecutor(/^(diffusion-|addon-logging-diffusion$)/, "SD v2.1 1B Q8_0 cold-load is too heavy for Device Farm devices (OOM, 3+GB)"), new SkipExecutor( /^translation-bergamot-.+-cache-reload$/, "Server-side Bare code path, identical across platforms — desktop coverage is source of truth", @@ -396,7 +383,6 @@ export const executor = createExecutor({ new MobileVisionExecutor(resources), new DownloadExecutor(), new DelegatedInferenceExecutor(), - new MobileDiffusionExecutor(resources), new LifecycleExecutor(resources), new ConfigExecutor(), new MobileCancellationExecutor(resources), diff --git a/packages/sdk/e2e/tests/mobile/executors/diffusion-executor.ts b/packages/sdk/e2e/tests/mobile/executors/diffusion-executor.ts deleted file mode 100644 index 61d656ac9c..0000000000 --- a/packages/sdk/e2e/tests/mobile/executors/diffusion-executor.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { DiffusionExecutor as SharedDiffusionExecutor } from "../../shared/executors/diffusion-executor.js"; - -export class MobileDiffusionExecutor extends SharedDiffusionExecutor { - private imageAssets: Record | null = null; - - private async loadImageAssets() { - if (!this.imageAssets) { - // @ts-ignore - assets.ts is generated at consumer build time - const assets = await import("../../../../assets"); - this.imageAssets = assets.images; - } - return this.imageAssets!; - } - - private async resolveAssetBytes(assetModule: number): Promise { - // @ts-ignore - expo-asset is a peer dependency available in mobile context - const { Asset } = await import("expo-asset"); - const asset = Asset.fromModule(assetModule); - asset.downloaded = false; - await asset.downloadAsync(); - const uri: string = asset.localUri || asset.uri; - if (!uri) { - throw new Error(`Failed to resolve asset: ${asset.name ?? "unknown"}`); - } - const fileUri = uri.startsWith("file://") ? uri : `file://${uri}`; - // @ts-ignore - expo-file-system is a peer dependency available in mobile context - const { File } = await import("expo-file-system"); - return await new File(fileUri).bytes(); - } - - private async resolveImageByName(name: string): Promise { - const fileName = name.split("/").pop()!; - const images = await this.loadImageAssets(); - const assetModule = images[fileName]; - if (!assetModule) { - throw new Error(`Image file not found in assets: ${fileName}`); - } - return await this.resolveAssetBytes(assetModule); - } - - protected override async resolveParams( - p: Record, - ): Promise> { - const out: Record = { ...p }; - - if (p.init_image !== undefined) { - if (typeof p.init_image !== "string") { - throw new Error( - `init_image in test params must be a string filename, got: ${typeof p.init_image}`, - ); - } - out.init_image = await this.resolveImageByName(p.init_image); - } - - if (p.image !== undefined) { - if (typeof p.image !== "string") { - throw new Error( - `image in test params must be a string filename, got: ${typeof p.image}`, - ); - } - out.image = await this.resolveImageByName(p.image); - } - - if (p.init_images !== undefined) { - if ( - !Array.isArray(p.init_images) || - !p.init_images.every((v) => typeof v === "string") - ) { - throw new Error( - "init_images in test params must be a string[] of image filenames", - ); - } - out.init_images = await Promise.all( - (p.init_images as string[]).map((n) => this.resolveImageByName(n)), - ); - } - - return out; - } -} diff --git a/packages/sdk/e2e/tests/shared/executors/diffusion-executor.ts b/packages/sdk/e2e/tests/shared/executors/diffusion-executor.ts index bda9aae16e..773baa4af2 100644 --- a/packages/sdk/e2e/tests/shared/executors/diffusion-executor.ts +++ b/packages/sdk/e2e/tests/shared/executors/diffusion-executor.ts @@ -1,217 +1,207 @@ -// Diffusion executor import { diffusion, upscale, type DiffusionClientParams } from "@qvac/sdk"; import { ValidationHelpers, type TestResult, type Expectation, + type HandlerFn, + type ExtractTest, } from "@tetherto/qvac-test-suite"; import { AbstractModelExecutor } from "./abstract-model-executor.js"; -import { diffusionTests } from "../../diffusion-tests.js"; - -// Minimum byte-level divergence between fusion output and the same-seed txt2img -// baseline. SDK output is bit-exact at fixed seed (see seedReproducibility), so -// 1% is well above noise and catches a silent fallback when init_images is dropped. -const MIN_FUSION_DIVERGENCE_RATIO = 0.01; +import { + diffusionTests, + diffusionBasicTxt2img, + diffusionDefaultSize, + diffusionNegativePrompt, + diffusionCfgScale, + diffusionSamplerEulerA, + diffusionSamplerHeun, + diffusionSchedulerKarras, + diffusionSeedReproducibility, + diffusionBatchCount, + diffusionBasicImg2img, + diffusionImg2imgImgCfgScale, + diffusionImg2imgVsTxt2imgBaseline, + diffusionImg2imgInvalidStrength, + diffusionStreaming, + diffusionStreamingProgress, + diffusionStatsPresent, + diffusionFaAccepted, + diffusionFaDisabledAccepted, + diffusionFusionFlux2Basic, + diffusionEsrganUpscaleX4, + diffusionStandaloneUpscalerX4, + diffusionEmptyPrompt, +} from "../../diffusion-tests.js"; + +// Min byte divergence vs same-seed/prompt txt2img baseline. Output is +// bit-exact at fixed seed (see seedReproducibility), so 1% is well above +// noise and catches a silent fallback when init_image / init_images is +// dropped server-side. +const MIN_DIVERGENCE_RATIO = 0.01; + +// Rolling param shape across resolveParams → buildParams → diffusion(). +// Asset fields are a `string` filename before resolveParams() and `Uint8Array` +// after. Index signature is required so test-specific param literals are +// assignable at the framework dispatch boundary. +export interface DiffusionParams { + prompt: string; + negative_prompt?: string; + width?: number; + height?: number; + steps?: number; + cfg_scale?: number; + img_cfg_scale?: number; + guidance?: number; + sampling_method?: DiffusionClientParams["sampling_method"]; + scheduler?: DiffusionClientParams["scheduler"]; + seed?: number; + batch_count?: number; + vae_tiling?: boolean; + increase_ref_index?: boolean; + auto_resize_ref_image?: boolean; + lora?: string; + strength?: number; + upscale?: DiffusionClientParams["upscale"]; + init_image?: string | Uint8Array; + init_images?: string[] | Uint8Array[]; + image?: string | Uint8Array; + repeats?: number; + [key: string]: unknown; +} export class DiffusionExecutor extends AbstractModelExecutor { pattern = /^diffusion-/; - protected handlers = Object.fromEntries( - diffusionTests.map((test) => [test.testId, this.generic.bind(this)]), - ) as never; - - async execute( - testId: string, - context: unknown, - params: unknown, - expectation: unknown, - ): Promise { - if (testId === "diffusion-seed-reproducibility") { - return await this.seedReproducibility(params, expectation); - } - if (testId === "diffusion-streaming-progress") { - return await this.streamingProgress(params, expectation); - } - if (testId === "diffusion-stats-present") { - return await this.statsPresent(params, expectation); - } - if (testId === "diffusion-fa-loads-and-runs") { - return await this.diffusionFaAccepted(params, expectation); - } - if (testId === "diffusion-fa-disabled-loads-and-runs") { - return await this.diffusionFaDisabledAccepted(params, expectation); - } - if (testId === "diffusion-fusion-flux2-basic") { - return await this.fusionFlux2Basic(params, expectation); - } - if (testId === "diffusion-esrgan-upscale-x4") { - return await this.esrganUpscaleX4(params, expectation); - } - if (testId === "diffusion-standalone-upscaler-x4") { - return await this.standaloneUpscalerX4(params, expectation); - } - - const handler = (this.handlers as Record Promise>)[testId]; - if (handler) { - return await handler.call(this, params, expectation); - } - return { passed: false, output: `Unknown test: ${testId}` }; - } - - // mobile and desktop subclasses override this to handle filesystem differences - protected async resolveParams(p: Record): Promise> { + // Exhaustive testId → handler map. Most tests share `runBasic`; comparative + // and streaming cases have dedicated methods. The `Required<...>` annotation + // turns "missing handler" into a TS compile error. + protected handlers: Required<{ + [K in (typeof diffusionTests)[number]["testId"]]: HandlerFn< + ExtractTest + >; + }> = { + [diffusionBasicTxt2img.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionDefaultSize.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionNegativePrompt.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionCfgScale.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionSamplerEulerA.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionSamplerHeun.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionSchedulerKarras.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionBatchCount.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionBasicImg2img.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionImg2imgImgCfgScale.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionImg2imgInvalidStrength.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionStreaming.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionEmptyPrompt.testId]: this.runBasic.bind(this, "diffusion"), + [diffusionFaAccepted.testId]: this.runBasic.bind(this, "diffusion-fa"), + [diffusionFaDisabledAccepted.testId]: this.runBasic.bind(this, "diffusion-fa-disabled"), + [diffusionEsrganUpscaleX4.testId]: this.runBasic.bind(this, "diffusion-esrgan"), + [diffusionSeedReproducibility.testId]: this.seedReproducibility.bind(this), + [diffusionStreamingProgress.testId]: this.streamingProgress.bind(this), + [diffusionStatsPresent.testId]: this.statsPresent.bind(this), + [diffusionImg2imgVsTxt2imgBaseline.testId]: this.img2imgVsTxt2imgBaseline.bind(this), + [diffusionFusionFlux2Basic.testId]: this.fusionFlux2Basic.bind(this), + [diffusionStandaloneUpscalerX4.testId]: this.standaloneUpscalerX4.bind(this), + }; + + // Subclasses override this to resolve string filenames in init_image / + // init_images / image to Uint8Array bytes via their platform's filesystem. + protected async resolveParams(p: DiffusionParams): Promise { return p; } - protected buildParams( - modelId: string, - p: Record, - ): DiffusionClientParams { - const params: Omit = { - modelId, - prompt: p.prompt as string, - }; + // ----- handlers ----- - if (p.negative_prompt != null) params.negative_prompt = p.negative_prompt as string; - if (p.width != null) params.width = p.width as number; - if (p.height != null) params.height = p.height as number; - if (p.steps != null) params.steps = p.steps as number; - if (p.cfg_scale != null) params.cfg_scale = p.cfg_scale as number; - if (p.guidance != null) params.guidance = p.guidance as number; - if (p.sampling_method != null) params.sampling_method = p.sampling_method as DiffusionClientParams["sampling_method"]; - if (p.scheduler != null) params.scheduler = p.scheduler as DiffusionClientParams["scheduler"]; - if (p.seed != null) params.seed = p.seed as number; - if (p.batch_count != null) params.batch_count = p.batch_count as number; - if (p.vae_tiling != null) params.vae_tiling = p.vae_tiling as boolean; - if (p.increase_ref_index != null) params.increase_ref_index = p.increase_ref_index as boolean; - if (p.auto_resize_ref_image != null) params.auto_resize_ref_image = p.auto_resize_ref_image as boolean; - if (p.lora != null) params.lora = p.lora as string; - if (p.strength != null) params.strength = p.strength as number; - if (p.upscale != null) params.upscale = p.upscale as DiffusionClientParams["upscale"]; - - if (p.init_image != null && p.init_images != null) { - throw new Error( - "Test params cannot set both init_image and init_images (mutually exclusive).", - ); - } - - if (p.init_images != null) { - return { ...params, init_images: p.init_images as Uint8Array[] }; - } - if (p.init_image != null) { - return { ...params, init_image: p.init_image as Uint8Array }; - } - return params; - } - - async generic(params: unknown, expectation: unknown): Promise { - const p = await this.resolveParams(params as Record); - const modelId = await this.resources.ensureLoaded("diffusion"); + // Unified path for every test that just runs diffusion() and validates + // outputs against the expectation. `resourceKey` selects which preconfigured + // model to load (FLUX.2 Klein, FLUX.2 with FA, SD+ESRGAN, ...). + async runBasic( + resourceKey: string, + params: DiffusionParams, + expectation: Expectation, + ): Promise { + const p = await this.resolveParams(params); + const modelId = await this.resources.ensureLoaded(resourceKey); try { - const genParams = this.buildParams(modelId, p); - const { outputs } = diffusion(genParams); + const { outputs } = diffusion(this.buildParams(modelId, p)); const buffers = await outputs; - return ValidationHelpers.validate(buffers, expectation as Expectation); + return ValidationHelpers.validate(buffers, expectation); } catch (error) { const errorMsg = error instanceof Error ? error.message : String(error); - const exp = expectation as Expectation; - if (exp.validation === "throws-error") { - return ValidationHelpers.validate(errorMsg, exp); + if (expectation.validation === "throws-error") { + return ValidationHelpers.validate(errorMsg, expectation); } return { passed: false, output: `Diffusion failed: ${errorMsg}` }; } } - async seedReproducibility( - params: unknown, - _expectation: unknown, - ): Promise { - const p = params as Record; + async seedReproducibility(params: DiffusionParams): Promise { const modelId = await this.resources.ensureLoaded("diffusion"); try { - const genParams = this.buildParams(modelId, p); - - const { outputs: outputs1 } = diffusion(genParams); - const buffers1 = await outputs1; + const genParams = this.buildParams(modelId, params); - const { outputs: outputs2 } = diffusion(genParams); - const buffers2 = await outputs2; + const buffers1 = await diffusion(genParams).outputs; + const buffers2 = await diffusion(genParams).outputs; if (buffers1.length === 0 || buffers2.length === 0) { return { passed: false, output: "No outputs generated" }; } - const match = - buffers1[0]!.length === buffers2[0]!.length && - buffers1[0]!.every((byte: number, i: number) => byte === buffers2[0]![i]); + const a = buffers1[0]!; + const b = buffers2[0]!; + const match = a.length === b.length && a.every((byte, i) => byte === b[i]); return { passed: match, output: match ? "Same seed produces identical output" - : `Outputs differ: ${buffers1[0]!.length} vs ${buffers2[0]!.length} bytes`, + : `Outputs differ: ${a.length} vs ${b.length} bytes`, }; } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { - passed: false, - output: `Seed reproducibility failed: ${errorMsg}`, - }; + return this.fail("Seed reproducibility", error); } } - async streamingProgress( - params: unknown, - _expectation: unknown, - ): Promise { - const p = params as Record; + async streamingProgress(params: DiffusionParams): Promise { const modelId = await this.resources.ensureLoaded("diffusion"); try { - const genParams = this.buildParams(modelId, p); - const { progressStream, outputs, stats } = diffusion(genParams); + const { progressStream, outputs, stats } = diffusion( + this.buildParams(modelId, params), + ); - const progressTicks: { step: number; totalSteps: number; elapsedMs: number }[] = []; - for await (const tick of progressStream) { - progressTicks.push(tick); - } + const ticks: { step: number; totalSteps: number; elapsedMs: number }[] = []; + for await (const tick of progressStream) ticks.push(tick); const buffers = await outputs; const finalStats = await stats; const hasOutputs = buffers.length > 0; const hasStats = finalStats != null; - const hasProgress = progressTicks.length > 0; - const progressValid = progressTicks.every( - (t) => typeof t.step === "number" && typeof t.totalSteps === "number" && typeof t.elapsedMs === "number", + const hasProgress = ticks.length > 0; + const progressValid = ticks.every( + (t) => + typeof t.step === "number" && + typeof t.totalSteps === "number" && + typeof t.elapsedMs === "number", ); return { passed: hasOutputs && hasStats && hasProgress && progressValid, - output: `Received ${buffers.length} output(s), ${progressTicks.length} progress tick(s), stats: ${hasStats ? "present" : "missing"}, progress valid: ${progressValid}`, + output: `Received ${buffers.length} output(s), ${ticks.length} progress tick(s), stats: ${hasStats ? "present" : "missing"}, progress valid: ${progressValid}`, }; } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { - passed: false, - output: `Streaming progress failed: ${errorMsg}`, - }; + return this.fail("Streaming progress", error); } } - async statsPresent( - params: unknown, - _expectation: unknown, - ): Promise { - const p = params as Record; + async statsPresent(params: DiffusionParams): Promise { const modelId = await this.resources.ensureLoaded("diffusion"); try { - const genParams = this.buildParams(modelId, p); - const { outputs, stats } = diffusion(genParams); - + const { outputs, stats } = diffusion(this.buildParams(modelId, params)); await outputs; const finalStats = await stats; @@ -219,173 +209,217 @@ export class DiffusionExecutor extends AbstractModelExecutor { + if (params.seed === undefined) { return { - passed: hasExpectedFields, - output: `Stats present: ${JSON.stringify(finalStats)}`, + passed: false, + output: "img2img-vs-txt2img test requires a fixed seed to compare against baseline", }; - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { passed: false, output: `Stats test failed: ${errorMsg}` }; } + if (params.init_image === undefined) { + return { passed: false, output: "img2img-vs-txt2img test requires init_image" }; + } + return this.compareWithBaseline(params, "init_image", "img2img"); } - async diffusionFaAccepted(params: unknown, expectation: unknown): Promise { - const p = await this.resolveParams(params as Record); - const modelId = await this.resources.ensureLoaded("diffusion-fa"); + // Compare fusion vs same-seed/prompt baseline with init_images dropped. + // If the addon silently ignores init_images, byte delta collapses and this fails. + async fusionFlux2Basic(params: DiffusionParams): Promise { + if (params.seed === undefined) { + return { + passed: false, + output: "fusion test requires a fixed seed to compare against baseline", + }; + } + return this.compareWithBaseline(params, "init_images", "Fusion"); + } + + async standaloneUpscalerX4( + params: DiffusionParams, + expectation: Expectation, + ): Promise { + const p = await this.resolveParams(params); + const modelId = await this.resources.ensureLoaded("upscaler"); + + if (!(p.image instanceof Uint8Array)) { + return { passed: false, output: "Standalone upscaler test requires image bytes" }; + } try { - const genParams = this.buildParams(modelId, p); - const { outputs } = diffusion(genParams); + const { outputs } = upscale({ + modelId, + image: p.image, + ...(p.repeats !== undefined && { repeats: p.repeats }), + }); const buffers = await outputs; - return ValidationHelpers.validate(buffers, expectation as Expectation); + return ValidationHelpers.validate(buffers, expectation); } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { passed: false, output: `diffusion_fa test failed: ${errorMsg}` }; + return this.fail("Standalone upscaler", error); } } - async diffusionFaDisabledAccepted(params: unknown, expectation: unknown): Promise { - const p = await this.resolveParams(params as Record); - const modelId = await this.resources.ensureLoaded("diffusion-fa-disabled"); + // ----- private helpers ----- - try { - const genParams = this.buildParams(modelId, p); - const { outputs } = diffusion(genParams); - const buffers = await outputs; - return ValidationHelpers.validate(buffers, expectation as Expectation); - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { passed: false, output: `diffusion_fa disabled test failed: ${errorMsg}` }; + // Translate the rolling DiffusionParams shape into the SDK's strict request + // type. All SDK fields are passed through verbatim; init_image / init_images + // must be Uint8Array by the time we reach here (resolveParams converts them). + protected buildParams( + modelId: string, + p: DiffusionParams, + ): DiffusionClientParams { + const params: Omit = { + modelId, + prompt: p.prompt, + }; + + if (p.negative_prompt !== undefined) params.negative_prompt = p.negative_prompt; + if (p.width !== undefined) params.width = p.width; + if (p.height !== undefined) params.height = p.height; + if (p.steps !== undefined) params.steps = p.steps; + if (p.cfg_scale !== undefined) params.cfg_scale = p.cfg_scale; + if (p.img_cfg_scale !== undefined) params.img_cfg_scale = p.img_cfg_scale; + if (p.guidance !== undefined) params.guidance = p.guidance; + if (p.sampling_method !== undefined) params.sampling_method = p.sampling_method; + if (p.scheduler !== undefined) params.scheduler = p.scheduler; + if (p.seed !== undefined) params.seed = p.seed; + if (p.batch_count !== undefined) params.batch_count = p.batch_count; + if (p.vae_tiling !== undefined) params.vae_tiling = p.vae_tiling; + if (p.increase_ref_index !== undefined) params.increase_ref_index = p.increase_ref_index; + if (p.auto_resize_ref_image !== undefined) params.auto_resize_ref_image = p.auto_resize_ref_image; + if (p.lora !== undefined) params.lora = p.lora; + if (p.strength !== undefined) params.strength = p.strength; + if (p.upscale !== undefined) params.upscale = p.upscale; + + if (p.init_image !== undefined && p.init_images !== undefined) { + throw new Error( + "Test params cannot set both init_image and init_images (mutually exclusive).", + ); } + if (p.init_images !== undefined) { + if (!isUint8ArrayList(p.init_images)) { + throw new Error("init_images must be Uint8Array[] by the time it reaches buildParams"); + } + return { ...params, init_images: p.init_images }; + } + if (p.init_image !== undefined) { + if (!(p.init_image instanceof Uint8Array)) { + throw new Error("init_image must be Uint8Array by the time it reaches buildParams"); + } + return { ...params, init_image: p.init_image }; + } + return params; } - // Compare fusion vs same-seed/prompt baseline with init_images dropped. - // If the addon silently ignores init_images, byte delta collapses and this fails. - async fusionFlux2Basic( - params: unknown, - _expectation: unknown, + // Single-image vs multi-image baseline-divergence comparison. `dropField` + // names the input that gets nulled out for the baseline run; `label` is the + // human-readable name used in the result message. + private async compareWithBaseline( + params: DiffusionParams, + dropField: "init_image" | "init_images", + label: string, ): Promise { - const p = await this.resolveParams(params as Record); + const p = await this.resolveParams(params); const modelId = await this.resources.ensureLoaded("diffusion"); - if (p.seed === undefined) { - return { - passed: false, - output: "fusion test requires a fixed seed to compare against baseline", - }; - } - try { - const fusionParams = this.buildParams(modelId, p); + const variantParams = this.buildParams(modelId, p); const baselineParams = this.buildParams(modelId, { ...p, - init_images: undefined, + [dropField]: undefined, }); - const { outputs: fusionOutputs } = diffusion(fusionParams); - const fusionBuffers = await fusionOutputs; + const variant = await diffusion(variantParams).outputs; + const baseline = await diffusion(baselineParams).outputs; - const { outputs: baselineOutputs } = diffusion(baselineParams); - const baselineBuffers = await baselineOutputs; - - if (fusionBuffers.length === 0 || baselineBuffers.length === 0) { + if (variant.length === 0 || baseline.length === 0) { return { passed: false, - output: `Missing output(s): fusion=${fusionBuffers.length}, baseline=${baselineBuffers.length}`, + output: `Missing output(s): ${label}=${variant.length}, baseline=${baseline.length}`, }; } - const diffRatio = this.byteDiffRatio( - fusionBuffers[0]!, - baselineBuffers[0]!, - ); - const passed = diffRatio > MIN_FUSION_DIVERGENCE_RATIO; - const deltaPct = (diffRatio * 100).toFixed(2); + // Guard against silent dimension mismatch (e.g. backend honoring + // init_image dims instead of requested width/height). PNG byte length + // varies by content/compression, so IHDR width/height is the only + // reliable invariant for cross-output comparison. + const dimErr = this.assertEqualPngDimensions(variant[0]!, baseline[0]!); + if (dimErr) return dimErr; + + const diff = this.byteDiffRatio(variant[0]!, baseline[0]!); + const passed = diff > MIN_DIVERGENCE_RATIO; + const deltaPct = (diff * 100).toFixed(2); return { passed, output: passed - ? `Fusion output differs from txt2img baseline (${deltaPct}% byte delta)` - : `Fusion output matches txt2img baseline too closely (${deltaPct}% byte delta)`, + ? `${label} output differs from txt2img baseline (${deltaPct}% byte delta)` + : `${label} output matches txt2img baseline too closely (${deltaPct}% byte delta) — ${dropField} likely dropped server-side`, }; } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { - passed: false, - output: `FLUX.2 fusion comparison failed: ${errorMsg}`, - }; + return this.fail(`${label} comparison`, error); } } - // Output dimensions are validated by the function expectation on the test - // definition (see diffusion-tests.ts → validateEsrganUpscale). - async esrganUpscaleX4( - params: unknown, - expectation: unknown, - ): Promise { - const p = await this.resolveParams(params as Record); - const modelId = await this.resources.ensureLoaded("diffusion-esrgan"); - - try { - const genParams = this.buildParams(modelId, p); - const { outputs } = diffusion(genParams); - const buffers = await outputs; - return ValidationHelpers.validate(buffers, expectation as Expectation); - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { passed: false, output: `ESRGAN upscale failed: ${errorMsg}` }; - } + private fail(label: string, error: unknown): TestResult { + const msg = error instanceof Error ? error.message : String(error); + return { passed: false, output: `${label} failed: ${msg}` }; } - async standaloneUpscalerX4( - params: unknown, - expectation: unknown, - ): Promise { - const p = await this.resolveParams(params as Record); - const modelId = await this.resources.ensureLoaded("upscaler"); - const image = p.image; + // PNG dimensions live in the IHDR chunk: bytes 16..23 of the file (8-byte + // signature + 4-byte length + 4-byte "IHDR" + 4 width + 4 height). + private readPngDims( + buf: Uint8Array, + ): { width: number; height: number } | null { + if (buf.length < 24) return null; + if (buf[0] !== 0x89 || buf[1] !== 0x50 || buf[2] !== 0x4e || buf[3] !== 0x47) return null; + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + return { width: view.getUint32(16, false), height: view.getUint32(20, false) }; + } - if (!(image instanceof Uint8Array)) { + private assertEqualPngDimensions( + left: Uint8Array, + right: Uint8Array, + ): TestResult | null { + const l = this.readPngDims(left); + const r = this.readPngDims(right); + if (!l || !r) { + return { passed: false, output: "One of the outputs is not a valid PNG" }; + } + if (l.width !== r.width || l.height !== r.height) { return { passed: false, - output: "Standalone upscaler test requires image bytes", + output: `Output dimensions mismatch: ${l.width}x${l.height} vs ${r.width}x${r.height} — comparison is only meaningful at equal dimensions`, }; } - - try { - const { outputs } = upscale({ - modelId, - image, - ...(p.repeats !== undefined && { repeats: p.repeats as number }), - }); - const buffers = await outputs; - return ValidationHelpers.validate(buffers, expectation as Expectation); - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { passed: false, output: `Standalone upscaler failed: ${errorMsg}` }; - } + return null; } private byteDiffRatio(left: Uint8Array, right: Uint8Array): number { const maxLength = Math.max(left.length, right.length); - if (maxLength === 0) { - return 0; - } + if (maxLength === 0) return 0; const minLength = Math.min(left.length, right.length); let changed = Math.abs(left.length - right.length); - for (let i = 0; i < minLength; i++) { - if (left[i] !== right[i]) { - changed++; - } + if (left[i] !== right[i]) changed++; } - return changed / maxLength; } } + +function isUint8ArrayList(value: unknown): value is Uint8Array[] { + return Array.isArray(value) && value.every((v) => v instanceof Uint8Array); +}