diff --git a/packages/lib-infer-diffusion/CHANGELOG.md b/packages/lib-infer-diffusion/CHANGELOG.md index 7fd8e6cdcb..a1e6c63100 100644 --- a/packages/lib-infer-diffusion/CHANGELOG.md +++ b/packages/lib-infer-diffusion/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.4.0] - 2026-04-21 + +### Added + +- Add LoRA support to diffusion generation via `run({ lora })`, forwarding a LoRA adapter path through the JS bridge and native addon into stable-diffusion.cpp's `sd_img_gen_params_t.loras` runtime path. +- Add a real LoRA integration test that downloads a compatible SD2.1 LoRA adapter, runs image generation with it, and verifies a valid PNG output is produced. + ## [0.3.0] - 2026-04-15 This release migrates the diffusion addon off `BaseInference` inheritance and onto the composable `createJobHandler` + `exclusiveRunQueue` utilities from `@qvac/infer-base@^0.4.0`. The constructor signature is replaced with a single object whose `files` field carries absolute paths for every model component, mirroring the parallel embed and LLM addon refactors. This is a breaking change — every caller must update. diff --git a/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.cpp b/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.cpp index 51bbbf8b33..8bd54e5691 100644 --- a/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.cpp +++ b/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.cpp @@ -168,6 +168,10 @@ const SdGenHandlersMap SD_GEN_HANDLERS = { [](SdGenConfig& c, const picojson::value& v) { c.negativePrompt = requireStr(v, "negative_prompt"); }}, + {"lora", + [](SdGenConfig& c, const picojson::value& v) { + c.loraPath = requireStr(v, "lora"); + }}, // ── Image dimensions // ──────────────────────────────────────────────────────── diff --git a/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.hpp b/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.hpp index c234ea2c0e..d3b78c49b0 100644 --- a/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.hpp +++ b/packages/lib-infer-diffusion/addon/src/handlers/SdGenHandlers.hpp @@ -26,6 +26,7 @@ struct SdGenConfig { // ── Prompt ──────────────────────────────────────────────────────────────── std::string prompt; std::string negativePrompt; + std::string loraPath; // ── Image dimensions ───────────────────────────────────────────────────── int width = 512; // must be a positive multiple of 8 diff --git a/packages/lib-infer-diffusion/addon/src/model-interface/SdModel.cpp b/packages/lib-infer-diffusion/addon/src/model-interface/SdModel.cpp index 70013f4373..eb55ce4daf 100644 --- a/packages/lib-infer-diffusion/addon/src/model-interface/SdModel.cpp +++ b/packages/lib-infer-diffusion/addon/src/model-interface/SdModel.cpp @@ -259,6 +259,31 @@ class SdImageBatch { const int count_; }; +struct PreparedLoras { + std::vector paths; + std::vector items; +}; + +// Mirrors the pinned fork's CLI flow in examples/common/common.hpp: +// build owned path storage first, then build sd_lora_t entries that point +// at that stable storage for the lifetime of generate_image(). +PreparedLoras prepareLoras(const std::string& loraPath) { + PreparedLoras prepared; + if (loraPath.empty()) { + return prepared; + } + + prepared.paths.push_back(loraPath); + + sd_lora_t item{}; + item.is_high_noise = false; + item.multiplier = 1.0f; + item.path = prepared.paths.back().c_str(); + prepared.items.push_back(item); + + return prepared; +} + } // namespace // --------------------------------------------------------------------------- @@ -484,6 +509,10 @@ std::any SdModel::process(const std::any& input) { sd_img_gen_params_t genParams{}; sd_img_gen_params_init(&genParams); + PreparedLoras loras = prepareLoras(gen.loraPath); + + genParams.loras = loras.items.empty() ? nullptr : loras.items.data(); + genParams.lora_count = static_cast(loras.items.size()); genParams.prompt = gen.prompt.c_str(); genParams.negative_prompt = gen.negativePrompt.c_str(); genParams.width = gen.width; diff --git a/packages/lib-infer-diffusion/index.d.ts b/packages/lib-infer-diffusion/index.d.ts index 2bb1b606b0..3ac6ba6d0c 100644 --- a/packages/lib-infer-diffusion/index.d.ts +++ b/packages/lib-infer-diffusion/index.d.ts @@ -151,6 +151,8 @@ export interface ImgStableDiffusionArgs { export interface GenerationParams { prompt: string negative_prompt?: string + /** Non-empty absolute path to a LoRA adapter (.safetensors, etc.) */ + lora?: string width?: number height?: number steps?: number diff --git a/packages/lib-infer-diffusion/index.js b/packages/lib-infer-diffusion/index.js index b4f7a18a14..aad455bc34 100644 --- a/packages/lib-infer-diffusion/index.js +++ b/packages/lib-infer-diffusion/index.js @@ -216,6 +216,7 @@ class ImgStableDiffusion { * @param {number} [params.batch_count=1] - Images per call * @param {boolean} [params.vae_tiling=false] - Enable VAE tiling (for large images) * @param {string} [params.cache_preset] - Cache preset: slow/medium/fast/ultra + * @param {string} [params.lora] - Non-empty absolute path to a LoRA adapter (.safetensors, etc.) * @param {Uint8Array} [params.init_image] - Source image bytes for img2img (PNG/JPEG). * FLUX2: in-context conditioning (ref_images). * Others: SDEdit (init_image + strength). @@ -235,6 +236,15 @@ class ImgStableDiffusion { ) } + if (params.lora != null) { + if (typeof params.lora !== 'string' || params.lora.length === 0) { + throw new TypeError('params.lora must be a non-empty string') + } + if (!path.isAbsolute(params.lora)) { + throw new TypeError(`params.lora must be an absolute path (got: ${params.lora})`) + } + } + // FLUX models require an explicit prediction type for img2img. // The C++ addon auto-detects the model family at load time, but // SdModel::process() only enters the FLUX ref_images path when diff --git a/packages/lib-infer-diffusion/package.json b/packages/lib-infer-diffusion/package.json index 2ea885a8b8..177904a398 100644 --- a/packages/lib-infer-diffusion/package.json +++ b/packages/lib-infer-diffusion/package.json @@ -1,6 +1,6 @@ { "name": "@qvac/diffusion-cpp", - "version": "0.3.0", + "version": "0.4.0", "description": "stable-diffusion.cpp addon for qvac image/video generation", "addon": true, "scripts": { diff --git a/packages/lib-infer-diffusion/scripts/generate-mobile-integration-tests.js b/packages/lib-infer-diffusion/scripts/generate-mobile-integration-tests.js index f72b2306de..c7eeb677b1 100644 --- a/packages/lib-infer-diffusion/scripts/generate-mobile-integration-tests.js +++ b/packages/lib-infer-diffusion/scripts/generate-mobile-integration-tests.js @@ -49,6 +49,14 @@ function buildFileContents (files) { } } + lines.push('') + lines.push('module.exports = {') + for (let i = 0; i < files.length; i++) { + const fnName = toFunctionName(files[i]) + lines.push(` ${fnName}${i < files.length - 1 ? ',' : ''}`) + } + lines.push('}') + return `${lines.join('\n')}\n` } diff --git a/packages/lib-infer-diffusion/test/integration/input-validation.test.js b/packages/lib-infer-diffusion/test/integration/input-validation.test.js index 6f3bae3d57..e70838e786 100644 --- a/packages/lib-infer-diffusion/test/integration/input-validation.test.js +++ b/packages/lib-infer-diffusion/test/integration/input-validation.test.js @@ -84,6 +84,71 @@ test('readImageDimensions | unrecognised format returns null', async (t) => { t.is(readImageDimensions(gif), null, 'GIF buffer returns null') }) +// ---------- LoRA path validation ---------- + +test('run | throws when lora is an empty string', async (t) => { + const model = new ImgStableDiffusion({ + files: { + model: '/tmp/stable-diffusion-v2-1-Q4_0.gguf' + }, + config: { threads: 1 }, + logger: console + }) + + try { + await model.run({ prompt: 'test', lora: '' }) + t.fail('should have thrown') + } catch (err) { + t.ok(err instanceof TypeError, 'throws TypeError') + t.ok( + /params\.lora must be a non-empty string/.test(err.message), + 'error message explains lora must be non-empty' + ) + } +}) + +test('run | throws when lora is not a string', async (t) => { + const model = new ImgStableDiffusion({ + files: { + model: '/tmp/stable-diffusion-v2-1-Q4_0.gguf' + }, + config: { threads: 1 }, + logger: console + }) + + try { + await model.run({ prompt: 'test', lora: 42 }) + t.fail('should have thrown') + } catch (err) { + t.ok(err instanceof TypeError, 'throws TypeError') + t.ok( + /params\.lora must be a non-empty string/.test(err.message), + 'error message explains lora must be a string' + ) + } +}) + +test('run | throws when lora is a relative path', async (t) => { + const model = new ImgStableDiffusion({ + files: { + model: '/tmp/stable-diffusion-v2-1-Q4_0.gguf' + }, + config: { threads: 1 }, + logger: console + }) + + try { + await model.run({ prompt: 'test', lora: 'adapter.safetensors' }) + t.fail('should have thrown') + } catch (err) { + t.ok(err instanceof TypeError, 'throws TypeError') + t.ok( + /params\.lora must be an absolute path/.test(err.message), + 'error message explains lora must be absolute' + ) + } +}) + // ---------- FLUX img2img prediction guard ---------- test('FLUX img2img | throws when prediction is omitted', async (t) => { diff --git a/packages/lib-infer-diffusion/test/integration/lora-bridge.test.js b/packages/lib-infer-diffusion/test/integration/lora-bridge.test.js new file mode 100644 index 0000000000..5d000cd782 --- /dev/null +++ b/packages/lib-infer-diffusion/test/integration/lora-bridge.test.js @@ -0,0 +1,151 @@ +'use strict' + +const fs = require('bare-fs') +const path = require('bare-path') +const os = require('bare-os') +const proc = require('bare-process') +const test = require('brittle') +const binding = require('../../binding') +const ImgStableDiffusion = require('../../index') +const { + ensureModel, + detectPlatform, + setupJsLogger, + isPng +} = require('./utils') + +const platform = detectPlatform() +const isDarwinX64 = os.platform() === 'darwin' && os.arch() === 'x64' +const isLinuxArm64 = os.platform() === 'linux' && os.arch() === 'arm64' +const isMobile = os.platform() === 'ios' || os.platform() === 'android' +const noGpu = proc.env && proc.env.NO_GPU === 'true' +const useCpu = isDarwinX64 || isLinuxArm64 || noGpu +const skip = isMobile || noGpu + +const DEFAULT_MODEL = { + name: 'stable-diffusion-v2-1-Q8_0.gguf', + url: 'https://huggingface.co/gpustack/stable-diffusion-v2-1-GGUF/resolve/main/stable-diffusion-v2-1-Q8_0.gguf' +} + +const LORA_ADAPTER = { + name: 'pytorch_lora_weights-sd21-comfyui.safetensors', + url: 'https://huggingface.co/radames/sd-21-DPO-LoRA/resolve/main/pytorch_lora_weights-sd21-comfyui.safetensors' +} + +test('SD2.1 txt2img with LoRA — generates a valid PNG image', { timeout: 600000, skip }, async (t) => { + setupJsLogger(binding) + + const [downloadedModelName, modelDir] = await ensureModel({ + modelName: DEFAULT_MODEL.name, + downloadUrl: DEFAULT_MODEL.url + }) + + const [downloadedLoraName] = await ensureModel({ + modelName: LORA_ADAPTER.name, + downloadUrl: LORA_ADAPTER.url + }) + + console.log('\n' + '='.repeat(60)) + console.log('STABLE DIFFUSION 2.1 — LORA INTEGRATION TEST') + console.log('='.repeat(60)) + console.log(` Platform : ${platform}`) + console.log(` Model : ${downloadedModelName}`) + console.log(` LoRA : ${downloadedLoraName}`) + console.log(` Models dir: ${modelDir}`) + + const modelPath = path.join(modelDir, downloadedModelName) + const loraPath = path.join(modelDir, downloadedLoraName) + t.ok(fs.existsSync(modelPath), 'Model file exists on disk') + t.ok(fs.existsSync(loraPath), 'LoRA adapter exists on disk') + + const model = new ImgStableDiffusion({ + files: { + model: modelPath + }, + config: { + threads: 4, + device: useCpu ? 'cpu' : 'gpu', + prediction: 'v' // SD2.1 uses v-prediction + }, + logger: console + }) + + const images = [] + const progressTicks = [] + + try { + // ── Load ───────────────────────────────────────────────────────────────── + console.log('\n=== Loading model ===') + const tLoad = Date.now() + await model.load() + const loadMs = Date.now() - tLoad + console.log(`Loaded in ${(loadMs / 1000).toFixed(1)}s`) + t.ok(loadMs < 120000, `Model loaded within 120s (took ${(loadMs / 1000).toFixed(1)}s)`) + + // ── Generate ────────────────────────────────────────────────────────────── + console.log('\n=== Generating image with LoRA ===') + const tGen = Date.now() + + const response = await model.run({ + prompt: 'a bright red sports car parked on a street, clean background, high detail, studio lighting', + negative_prompt: 'blurry, low quality, watermark', + lora: loraPath, + steps: 1, + width: 512, + height: 512, + cfg_scale: 7.5, + seed: 42 // fixed seed for reproducibility + }) + + await response + .onUpdate((data) => { + if (data instanceof Uint8Array) { + images.push(data) + } else if (typeof data === 'string') { + try { + const tick = JSON.parse(data) + if ('step' in tick && 'total' in tick) { + progressTicks.push(tick) + } + } catch (_) {} + } + }) + .await() + + const genMs = Date.now() - tGen + console.log(`\nGenerated in ${(genMs / 1000).toFixed(1)}s`) + + // ── Assertions ──────────────────────────────────────────────────────────── + t.ok(progressTicks.length > 0, `Received progress ticks (got ${progressTicks.length})`) + t.is(images.length, 1, 'Received exactly 1 image') + + const img = images[0] + t.ok(img instanceof Uint8Array, 'Image is a Uint8Array') + t.ok(img.length > 0, `Image is non-empty (${img.length} bytes)`) + t.ok(isPng(img), 'Image has valid PNG magic bytes') + + // Save output for CI artifact upload — filename encodes test origin. + // Saved to modelDir so mobile has write permission to the same path. + const outPath = path.join(modelDir, 'generate-image--sd2-lora-txt2img-seed42.png') + fs.writeFileSync(outPath, img) + console.log(`\nSaved → ${outPath}`) + + // ── Summary ─────────────────────────────────────────────────────────────── + console.log('\n' + '='.repeat(60)) + console.log('TEST SUMMARY') + console.log('='.repeat(60)) + console.log(` Load time : ${(loadMs / 1000).toFixed(1)}s`) + console.log(` Gen time : ${(genMs / 1000).toFixed(1)}s`) + console.log(` Steps ticks : ${progressTicks.length}`) + console.log(` Image size : ${img.length} bytes`) + console.log(' PNG valid : true') + console.log('='.repeat(60)) + } finally { + console.log('\n=== Cleanup ===') + await model.unload() + try { + binding.releaseLogger() + } catch (_) {} + console.log('Done.') + } +}) diff --git a/packages/lib-infer-diffusion/test/mobile/integration.auto.cjs b/packages/lib-infer-diffusion/test/mobile/integration.auto.cjs index 3e86768bfd..118ecc31b1 100644 --- a/packages/lib-infer-diffusion/test/mobile/integration.auto.cjs +++ b/packages/lib-infer-diffusion/test/mobile/integration.auto.cjs @@ -38,6 +38,23 @@ async function runInputValidationTest (options = {}) { // eslint-disable-line no return runIntegrationModule('../integration/input-validation.test.js', options) } +async function runLoraBridgeTest (options = {}) { // eslint-disable-line no-unused-vars + return runIntegrationModule('../integration/lora-bridge.test.js', options) +} + async function runModelLoadingTest (options = {}) { // eslint-disable-line no-unused-vars return runIntegrationModule('../integration/model-loading.test.js', options) } + +module.exports = { + runApiBehaviorTest, + runGenerateImageFlux2I2iTest, + runGenerateImageFlux2Test, + runGenerateImageSd3I2iTest, + runGenerateImageSd3Test, + runGenerateImageSdxlTest, + runGenerateImageTest, + runInputValidationTest, + runLoraBridgeTest, + runModelLoadingTest +} diff --git a/packages/lib-infer-diffusion/test/unit/test_sd_gen_handlers.cpp b/packages/lib-infer-diffusion/test/unit/test_sd_gen_handlers.cpp index 2da8b4505a..4c460d6547 100644 --- a/packages/lib-infer-diffusion/test/unit/test_sd_gen_handlers.cpp +++ b/packages/lib-infer-diffusion/test/unit/test_sd_gen_handlers.cpp @@ -145,6 +145,11 @@ TEST(SdGenHandlers_Scheduler, UnknownSchedulerThrows) { StatusError); } +TEST(SdGenHandlers_Prompt, LoraPathMapsCorrectly) { + auto cfg = applyOne("lora", str("/tmp/test-lora.safetensors")); + EXPECT_EQ(cfg.loraPath, "/tmp/test-lora.safetensors"); +} + // ───────────────────────────────────────────────────────────────────────────── // 3. parseCacheMode // ─────────────────────────────────────────────────────────────────────────────