diff --git a/packages/cli/src/commands/render.test.ts b/packages/cli/src/commands/render.test.ts index bddca8486f..77d0049f34 100644 --- a/packages/cli/src/commands/render.test.ts +++ b/packages/cli/src/commands/render.test.ts @@ -193,6 +193,17 @@ vi.mock("../browser/preflight.js", () => ({ runEnvironmentChecks: vi.fn(async () => preflightState.result), })); +// The "render command explicit composition" test below drives the real +// `render.js` command handler, which takes the plan-based `execute.ts` path +// (not the `renderLocal` unit under test above) — that path calls +// `ensureBrowser` directly instead of going through the mocked preflight. +// Unmocked, it performs a real network download of chrome-headless-shell into +// the shared `~/.cache/hyperframes/chrome`, racing other packages' browser +// tests in CI. +vi.mock("../browser/manager.js", () => ({ + ensureBrowser: vi.fn(async () => ({ executablePath: "/mock/chrome", source: "cache" })), +})); + vi.mock("../utils/orphanCleanup.js", () => ({ killOrphanedProcesses: vi.fn(() => { orphanCleanupState.calls += 1; diff --git a/packages/core/src/runtime/audioFx.test.ts b/packages/core/src/runtime/audioFx.test.ts index 9774515810..09374e12e4 100644 --- a/packages/core/src/runtime/audioFx.test.ts +++ b/packages/core/src/runtime/audioFx.test.ts @@ -315,6 +315,124 @@ describe("attachElementFxChain", () => { }); }); + /** + * Lanes are committed to absolute context times, so the schedule is only + * right for the rate it was booked at. Bumping `playbackRate` alone left a + * lowpass sweeping over its original 10 wall-clock seconds while the audio + * underneath ran through 20 clip-seconds of material — and the runtime's + * stopAll()+reschedule recovery never fired for an unbounded source. + */ + describe("a rate change mid-playback", () => { + /** Records what was booked and when, without the browser's overlap rules. */ + class TimedParam { + curves: { time: number; duration: number }[] = []; + ramps: number[] = []; + value = 0; + setValueAtTime(v: number): void { + this.value = v; + } + linearRampToValueAtTime(v: number, t: number): void { + this.ramps.push(t); + this.value = v; + } + setValueCurveAtTime(_v: Float32Array, time: number, duration: number): void { + this.curves.push({ time, duration }); + } + cancelScheduledValues(): void {} + cancelAndHoldAtTime(): void {} + /** The last span booked, however the scheduler chose to express it. */ + last(): { time: number; duration: number } | undefined { + return this.curves.at(-1); + } + } + + const sweep = { + version: 1, + nodes: [{ type: "lowpass", id: "n1", params: { frequency: 300, q: 0.707 } }], + }; + const lane = JSON.stringify({ + version: 1, + lanes: [ + { + target: "fx.n1.frequency", + points: [ + { t: 0, v: 300 }, + { t: 8, v: 3000 }, + ], + }, + ], + }); + + const build = () => { + const clock = { currentTime: 0 }; + const made: { frequency: TimedParam }[] = []; + class TimedNode extends Node { + override frequency = new TimedParam() as unknown as { value: number }; + } + class TimedCtx extends Ctx { + get currentTime(): number { + return clock.currentTime; + } + override createBiquadFilter(): Node { + const n = new TimedNode(); + made.push(n as unknown as { frequency: TimedParam }); + return n; + } + } + const node = document.createElement("audio"); + node.setAttribute("data-fx-chain", JSON.stringify(sweep)); + node.setAttribute("data-automation", lane); + document.body.append(node); + const handle = attachElementFxChain( + new TimedCtx() as unknown as BaseAudioContext, + node, + new Node() as never, + new Node() as never, + { scheduledAt: 0, elapsed: 0, rate: 1 }, + ); + return { clock, node, handle, param: () => made[0]?.frequency as unknown as TimedParam }; + }; + + it("re-aims the envelope so the sweep still ends with the material", () => { + const { clock, handle, param } = build(); + // Booked at 1x: the whole 8 s lane spans 8 s of context time. + expect(param().last()).toEqual({ time: 0, duration: 8 }); + + clock.currentTime = 2; + handle?.setRate(2); + + // 6 clip-seconds are left, and at 2x they take 3 wall-clock seconds. + // Without this the sweep kept its original plan to t=8 while the audio + // ran out at t=5. + expect(param().last()).toEqual({ time: 2, duration: 3 }); + }); + + it("measures later edits from the new rate, not the one it started at", async () => { + // `elapsed` advances at whatever rate the reference frame holds, so a + // frame left at 1x re-aims every subsequent edit at the wrong clip + // position for as long as the track plays. + const { clock, node, handle, param } = build(); + clock.currentTime = 2; + handle?.setRate(2); + + clock.currentTime = 4; + // 2 wall-clock seconds at 2x is 4 clip-seconds, so the playhead is at 6 + // and 2 clip-seconds remain: 1 second of wall clock. + node.setAttribute("data-automation", lane); + await new Promise((r) => setTimeout(r, 0)); + expect(param().last()).toEqual({ time: 4, duration: 1 }); + }); + + it("ignores a rate that is not a rate", () => { + const { clock, handle, param } = build(); + clock.currentTime = 2; + handle?.setRate(0); + handle?.setRate(Number.NaN); + handle?.setRate(1); + expect(param().last()).toEqual({ time: 0, duration: 8 }); + }); + }); + it("tears the chain down on dispose", () => { const src = new Node(); const dst = new Node(); diff --git a/packages/core/src/runtime/audioFx.ts b/packages/core/src/runtime/audioFx.ts index 835aec965c..d2b6789767 100644 --- a/packages/core/src/runtime/audioFx.ts +++ b/packages/core/src/runtime/audioFx.ts @@ -93,15 +93,29 @@ export function readElementAutomation(el: { * first effect is then heard without rescheduling the source. * * With `timing`, the element's automation lanes are scheduled onto the built - * effects as AudioParam ramps, and rescheduled when the attribute is edited. + * effects as AudioParam ramps, and rescheduled when the attribute is edited or + * `setRate` reports the transport changed speed. */ +export interface ElementFxHandle { + dispose(): void; + /** + * Re-aim every booked envelope at a new playback rate. + * + * Lanes are committed to absolute context times, so a param scheduled at 1× + * keeps its original wall-clock plan while the audio underneath runs at the + * new speed: a lowpass sweeping over 10 clip-seconds, switched to 2×, eats + * 20 s of material in 10 s of wall clock with the sweep unchanged. + */ + setRate(rate: number): void; +} + export function attachElementFxChain( ctx: BaseAudioContext, el: { getAttribute?(name: string): string | null }, source: AudioNode, destination: AudioNode, timing?: AutomationTiming, -): { dispose(): void } | null { +): ElementFxHandle | null { const { chain } = readChain(el); // Null means the source runs straight into its gain: an empty chain, or one @@ -178,20 +192,26 @@ export function attachElementFxChain( at && handle ? scheduleChainAutomation(readAutomation(el, next), next, handle.nodes, at) : []; }; + // The reference frame every later reschedule measures from. Mutable because a + // rate change rebases it: `elapsed` has to stop advancing at the old rate the + // instant the new one takes effect, or every subsequent edit re-aims the + // envelope at the wrong clip position. + let frame: AutomationTiming | null = timing ? { ...timing } : null; + attach(chain); - scheduleFor(chain, timing ?? null); + scheduleFor(chain, frame); /** * Re-aim the envelope at the live playhead. An edit lands mid-playback, so * the clip has advanced past the offset the source was scheduled with. */ const timingNow = (): AutomationTiming | null => { - if (!timing) return null; - const now = typeof ctx.currentTime === "number" ? ctx.currentTime : timing.scheduledAt; + if (!frame) return null; + const now = typeof ctx.currentTime === "number" ? ctx.currentTime : frame.scheduledAt; return { scheduledAt: now, - elapsed: timing.elapsed + (now - timing.scheduledAt) * timing.rate, - rate: timing.rate, + elapsed: frame.elapsed + (now - frame.scheduledAt) * frame.rate, + rate: frame.rate, }; }; @@ -251,6 +271,15 @@ export function attachElementFxChain( } return { + setRate: (rate: number) => { + const at = timingNow(); + if (disposed || !at || !Number.isFinite(rate) || rate <= 0 || rate === at.rate) return; + // Rebased at the playhead the OLD rate carried us to, then replayed from + // there at the new one. + frame = { ...at, rate }; + cancelParamLane(automated, at.scheduledAt); + scheduleFor(readChain(el).chain, frame); + }, dispose: () => { disposed = true; observer?.disconnect(); diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index f1597ccc8b..35f935bc5a 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -289,6 +289,23 @@ describe("WebAudioTransport", () => { expect(mock.sourceNode.playbackRate.value).toBe(2); }); + it("setRate re-aims each source's FX automation, not just its playback rate", async () => { + // The lanes are committed to absolute context times when the source is + // scheduled, so bumping playbackRate alone left every automated parameter + // running its original plan over audio moving at a different speed. + const { transport, mock, gen } = setupTransport(100); + await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1); + const active = (transport as unknown as { _activeSources: { fx?: unknown }[] }) + ._activeSources; + const setRate = vi.fn(); + active[0]!.fx = { dispose: vi.fn(), setRate }; + + transport.setRate(2); + + expect(setRate).toHaveBeenCalledWith(2); + expect(mock.sourceNode.playbackRate.value).toBe(2); + }); + it("setRate before any sources are scheduled does not throw", () => { const transport = new WebAudioTransport(); expect(() => transport.setRate(2)).not.toThrow(); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index 07b1af27c7..e2e2dc66ed 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -1,4 +1,4 @@ -import { attachElementFxChain, readElementAutomation } from "./audioFx.js"; +import { attachElementFxChain, readElementAutomation, type ElementFxHandle } from "./audioFx.js"; import { scheduleParamLane, volumeLane, @@ -82,7 +82,7 @@ export type ScheduledSource = { sourceNode: AudioBufferSourceNode; gainNode: GainNode; /** FX chain spliced between source and gain, when the element carries one. */ - fx?: { dispose(): void } | null; + fx?: ElementFxHandle | null; compositionStart: number; mediaStart: number; scheduledAt: number; @@ -295,6 +295,14 @@ export class WebAudioTransport { * `getTime()` stays continuous across the change. Sources scheduled to * start in the future keep their original wallclock start time — callers * that need rate-correct future starts should `stopAll()` and reschedule. + * + * Each source's FX automation is re-aimed too. Lanes are committed to + * absolute context times when the source is scheduled, so bumping only + * `playbackRate` left every automated parameter running its original plan + * over audio moving at a different speed. The `stopAll()`+reschedule recovery + * in the runtime is no help here: it only fires for bounded sources, and a + * project-level music bed with no `data-duration` is unbounded, so it never + * recovered at all. */ setRate(rate: number): boolean { const safeRate = normalizeRate(rate); @@ -307,6 +315,7 @@ export class WebAudioTransport { for (const source of this._activeSources) { try { source.sourceNode.playbackRate.value = safeRate; + source.fx?.setRate(safeRate); } catch (err) { swallow("webAudioTransport.setRate", err); } diff --git a/packages/engine/src/services/audioFxRender.test.ts b/packages/engine/src/services/audioFxRender.test.ts index b87fbb4e6b..29282dca12 100644 --- a/packages/engine/src/services/audioFxRender.test.ts +++ b/packages/engine/src/services/audioFxRender.test.ts @@ -180,7 +180,7 @@ describe("applyAudioFxChain", () => { join(dir, "out.wav"), { trackId: "t" }, ); - expect(out).toBe(input); + expect(out).toEqual({ path: input, envelopeBaked: false }); expect(existsSync(join(dir, "out.wav"))).toBe(false); }); @@ -213,7 +213,7 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { outPath, { trackId: "t" }, ); - expect(result).toBe(outPath); + expect(result).toEqual({ path: outPath, envelopeBaked: false }); const before = readWav(input).samples; const after = readWav(outPath).samples; expect(after.length).toBe(before.length); @@ -283,6 +283,139 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => { expect(tail).toBeGreaterThan(head + 15); }, 180_000); + it("ducks a hot chain before quantising it, not after", async () => { + // A +6 dB peaking band on a 0.9 tone leaves the chain output around 1.8 — + // well past full scale — and the volume lane immediately halves it. Baking + // the envelope into the float samples lands that at ~0.9 intact. Letting + // writeWav clamp first and ducking the file afterwards lands at ~0.5 with + // the tops sheared off: distortion the render bakes in and preview, which + // is float all the way through, never has. + // + // Stereo with a step partway through, so the same run also proves the + // envelope is walked per frame across all channels rather than per channel: + // one walker restarted for a second plane would hand it the tail gain for + // its whole length. + const input = join(dir, "hot-in.wav"); + const frames = Math.floor(SR * 0.3); + const s = new Float32Array(frames * 2); + for (let i = 0; i < frames; i++) { + const v = 0.9 * Math.sin((2 * Math.PI * 440 * i) / SR); + s[i * 2] = v; + s[i * 2 + 1] = v; + } + writeWav(input, s, SR, 2); + + const outPath = join(dir, "hot-out.wav"); + const result = await applyAudioFxChain( + input, + { + version: 1, + nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: 6, q: 1 } }], + }, + outPath, + { + trackId: "t", + envelope: { + // 0.5 for the first 150 ms, then 0.25 — the step is a segment + // advance, which is what the walker's cursor exists for. + keyframes: [ + { time: 0, volume: 0.5 }, + { time: 0.15, volume: 0.5 }, + { time: 0.1501, volume: 0.25 }, + { time: 0.3, volume: 0.25 }, + ], + trackStart: 0, + baseVolume: 1, + }, + }, + ); + expect(result.envelopeBaked).toBe(true); + + const out = readWav(outPath); + expect(out.channels).toBe(2); + const channel = (c: number): Float32Array => + Float32Array.from({ length: frames }, (_, i) => out.samples[i * 2 + c] ?? 0); + const window = (s: Float32Array, from: number, to: number): Float32Array => + s.slice(Math.floor(from * SR), Math.floor(to * SR)); + + for (const c of [0, 1]) { + const plane = channel(c); + // Past the filter's settling transient, before the step. + const loud = window(plane, 0.1, 0.14); + const peak = Math.max(...Array.from(loud, Math.abs)); + // ~0.9. Clamped-then-ducked gives 0.5; a dropped envelope gives 1.0; a + // walker restarted per plane gives channel 1 the 0.25 tail gain. + expect(peak).toBeGreaterThan(0.8); + expect(peak).toBeLessThan(0.95); + // And still a sine, not a squared-off one: clipping 1.8 down to 1.0 pulls + // the crest factor from 1.41 towards 1.15. + expect(peak / rms(loud)).toBeGreaterThan(1.35); + // The step landed, so the envelope was sampled over time, not once. + const quiet = window(plane, 0.2, 0.29); + expect(Math.max(...Array.from(quiet, Math.abs))).toBeCloseTo(peak / 2, 1); + } + }, 180_000); + + it("round-trips a track larger than one transfer chunk", async () => { + // The PCM used to cross in a single evaluate pair, so a stereo track past + // ~8.7 minutes blew puppeteer's 256 MB frame cap and failed the render. + // It now goes a chunk at a time; this clip is 45 s stereo, so each plane + // spans two chunks and the seam is inside the audio rather than at its end. + // + // A ramp, not a tone: every sample is a unique position marker, so a chunk + // dropped, reordered or over-read shows up as a value at the wrong place + // instead of hiding inside a periodic signal. + const input = join(dir, "long-in.wav"); + const frames = SR * 45; + const at = (i: number): number => -0.9 + (1.8 * i) / (frames - 1); + const s = new Float32Array(frames * 2); + for (let i = 0; i < frames; i++) { + s[i * 2] = at(i); + s[i * 2 + 1] = -at(i); + } + writeWav(input, s, SR, 2); + + const outPath = join(dir, "long-out.wav"); + await applyAudioFxChain( + input, + // Transparent: a peaking band at 0 dB is unity, so the output is the + // input and any difference is the transfer's doing. + { + version: 1, + nodes: [{ type: "peaking", enabled: true, params: { frequency: 1000, gain: 0, q: 1 } }], + }, + outPath, + { trackId: "t" }, + ); + + const out = readWav(outPath); + expect(out.channels).toBe(2); + // A dropped tail chunk shows up here first. + expect(out.samples.length).toBe(frames * 2); + + // Probe across the whole clip and tightly around the 8 MiB seam. + const seam = (8 * 1024 * 1024) / 4; + const probes = [ + ...Array.from({ length: 60 }, (_, k) => Math.floor((k * (frames - 1)) / 59)), + ...[-2, -1, 0, 1, 2].map((d) => seam + d), + ].filter((i) => i >= 0 && i < frames); + for (const i of probes) { + // Two 16-bit steps of tolerance: the input was quantised on the way in + // and the output again on the way out. + expect(out.samples[i * 2]).toBeCloseTo(at(i), 3); + expect(out.samples[i * 2 + 1]).toBeCloseTo(-at(i), 3); + } + + // The ramp only ever rises, so this reads every sample and fails on a + // chunk reordered, duplicated or over-read anywhere in the file — not just + // at the points probed above. + let breaks = 0; + for (let i = 1; i < frames; i++) { + if ((out.samples[i * 2] ?? 0) < (out.samples[(i - 1) * 2] ?? 0)) breaks += 1; + } + expect(breaks).toBe(0); + }, 180_000); + it("renders a multi-effect chain including reverb", async () => { const input = join(dir, "in.wav"); tone(input); @@ -311,7 +444,7 @@ describe("an empty track", () => { // — and without paying for a browser to decide it. await expect( applyAudioFxChain(input, chainOf("peaking"), output, { trackId: "t" }), - ).resolves.toBe(input); + ).resolves.toEqual({ path: input, envelopeBaked: false }); expect(existsSync(output)).toBe(false); }); }); diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts index b09fb1f853..950b5ce463 100644 --- a/packages/engine/src/services/audioFxRender.ts +++ b/packages/engine/src/services/audioFxRender.ts @@ -20,6 +20,8 @@ import { getAudioFxRuntimeScript } from "@hyperframes/core/audio-fx-runtime"; import { enabledAudioFxNodes, type HfAudioFxChain } from "@hyperframes/core/audio-fx"; import { serializeAutomation, type HfAutomation } from "@hyperframes/core/audio-automation"; import { acquireBrowser } from "./browserManager.js"; +import { createEnvelopeWalker } from "./audioVolumeEnvelope.js"; +import type { AudioVolumeKeyframe } from "./audioMixer.types.js"; export class AudioFxRenderError extends Error { constructor(message: string) { @@ -183,10 +185,66 @@ function interleave(planes: readonly Float32Array[]): Float32Array { return out; } +/** + * Bytes of PCM per CDP message. + * + * The whole track used to cross in a single `page.evaluate` pair — ~184 MB of + * base64 for a 3-minute stereo 48 kHz clip, in one WebSocket frame each way. + * puppeteer-core caps its frames at 256 MB and the browser pool does not use + * the pipe transport, so stereo past ~8.7 minutes failed the render outright; + * V8's max string length is a second wall not far beyond it. Chunking bounds + * both, and bounds the peak Node-side allocation with them: the mixer renders + * tracks concurrently, so every track's payload was live at once. + * + * 8 MiB encodes to ~11 MB of base64. A multiple of 4, so a chunk boundary + * never falls inside a float. + */ +const TRANSFER_BYTES = 8 * 1024 * 1024; + +/** The page-side handover buffers, named off `window` so each step can find them. */ +interface AudioFxPageIo { + in: Uint8Array[][]; + out: Float32Array[]; +} + +interface AudioFxWindow { + __HF_AUDIO_FX?: { + render(p: Float32Array[], r: number, c: string, a?: string): Promise; + }; + __HF_FX_IO?: AudioFxPageIo; +} + +/** + * Multiply every channel by the envelope, in place, one gain per frame. + * + * Frame-outer rather than plane-outer on purpose: the walker's cursor only + * moves forward, so restarting each channel at t=0 would hand the second one + * the tail gain for its whole length. + */ +function applyEnvelopeToPlanes( + planes: readonly Float32Array[], + sampleRate: number, + gainAt: (time: number) => number, +): void { + const frames = planes[0]?.length ?? 0; + for (let frame = 0; frame < frames; frame += 1) { + const gain = gainAt(frame / sampleRate); + for (const plane of planes) plane[frame] = (plane[frame] ?? 0) * gain; + } +} + /** * Run a chain over `inputWav`, writing `outputWav`. Resolves to the path to use - * downstream: `outputWav` when the chain did something, `inputWav` untouched - * when the chain was empty. + * downstream — `outputWav` when the chain did something, `inputWav` untouched + * when the chain was empty — plus whether the volume envelope was baked here. + * + * The envelope is applied to the float samples the chain produced, BEFORE + * `writeWav` quantises them. Leaving it to the mixer's second pass meant a + * chain that overshoots full scale was destructively clipped to ±1 and only + * then ducked, so a track the lane pulls 12 dB down still rendered the + * distortion — which preview, working in float throughout, never had. + * `writeWav`'s clamp stays: it is the correct last resort for a signal that is + * still hot after the duck. * * Failure is fatal to the caller rather than a soft per-track warning: quietly * rendering the dry signal ships a mix that sounds plausible and is not what @@ -196,9 +254,14 @@ export async function applyAudioFxChain( inputWav: string, chain: HfAudioFxChain, outputWav: string, - options: { trackId: string; signal?: AbortSignal; automation?: HfAutomation }, -): Promise { - if (enabledAudioFxNodes(chain).length === 0) return inputWav; + options: { + trackId: string; + signal?: AbortSignal; + automation?: HfAutomation; + envelope?: { keyframes: AudioVolumeKeyframe[]; trackStart: number; baseVolume: number }; + }, +): Promise<{ path: string; envelopeBaked: boolean }> { + if (enabledAudioFxNodes(chain).length === 0) return { path: inputWav, envelopeBaked: false }; if (!existsSync(inputWav)) { throw new AudioFxRenderError(`Audio FX input is missing: ${inputWav}`); } @@ -212,7 +275,7 @@ export async function applyAudioFxChain( // past the end of its source, so one mis-set `data-media-start` used to take // the render down. Guarded here as well as in the runtime so an empty track // never costs a browser. - if ((planes[0]?.length ?? 0) === 0) return inputWav; + if ((planes[0]?.length ?? 0) === 0) return { path: inputWav, envelopeBaked: false }; // Both resources are taken INSIDE the try that releases them. The lease used // to be acquired above it, with the mkdtemp between — so a failure there @@ -239,71 +302,119 @@ export async function applyAudioFxChain( await page.goto(pathToFileURL(hostPage).href, { waitUntil: "domcontentloaded" }); await page.addScriptTag({ content: getAudioFxRuntimeScript() }); - const rendered = (await page.evaluate( - async ([channelB64, rate, chainJson, automationJson]: [ - string[], - number, - string, - string, - ]) => { - const decode = (b64: string): Float32Array => { - const bin = atob(b64); - const bytes = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); - return new Float32Array(bytes.buffer); - }; - const api = ( - window as unknown as { - __HF_AUDIO_FX?: { - render( - p: Float32Array[], - r: number, - c: string, - a?: string, - ): Promise; - }; + // Hand the input over a chunk at a time. The chunks stay separate byte + // arrays page-side rather than being concatenated into one string, so + // neither the frame cap nor V8's string limit sees the whole track. + await page.evaluate((count: number) => { + (window as unknown as AudioFxWindow).__HF_FX_IO = { + in: Array.from({ length: count }, (): Uint8Array[] => []), + out: [], + }; + }, planes.length); + + for (let p = 0; p < planes.length; p += 1) { + const plane = planes[p]; + if (!plane) continue; + const bytes = Buffer.from(plane.buffer, plane.byteOffset, plane.length * 4); + for (let at = 0; at < bytes.length; at += TRANSFER_BYTES) { + await page.evaluate( + ([index, b64]: [number, string]) => { + const bin = atob(b64); + const chunk = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) chunk[i] = bin.charCodeAt(i); + (window as unknown as AudioFxWindow).__HF_FX_IO?.in[index]?.push(chunk); + }, + [p, bytes.subarray(at, at + TRANSFER_BYTES).toString("base64")] as [number, string], + ); + } + } + + const outLengths = (await page.evaluate( + async ([rate, chainJson, automationJson]: [number, string, string]) => { + const w = window as unknown as AudioFxWindow; + const io = w.__HF_FX_IO; + if (!w.__HF_AUDIO_FX || !io) throw new Error("audio FX runtime failed to load"); + const inPlanes = io.in.map((chunks) => { + const bytes = new Uint8Array(chunks.reduce((n, c) => n + c.length, 0)); + let at = 0; + for (const chunk of chunks) { + bytes.set(chunk, at); + at += chunk.length; } - ).__HF_AUDIO_FX; - if (!api) throw new Error("audio FX runtime failed to load"); - const out = await api.render( - channelB64.map(decode), + return new Float32Array(bytes.buffer); + }); + // Dropped before the render allocates its own buffers, so the page + // does not hold two copies of the track at once. + io.in = []; + io.out = await w.__HF_AUDIO_FX.render( + inPlanes, rate, chainJson, automationJson || undefined, ); - const encode = (plane: Float32Array): string => { - const u8 = new Uint8Array(plane.buffer, plane.byteOffset, plane.length * 4); - let s = ""; - const CHUNK = 0x8000; - for (let i = 0; i < u8.length; i += CHUNK) { - s += String.fromCharCode.apply(null, Array.from(u8.subarray(i, i + CHUNK))); - } - return btoa(s); - }; - return out.map(encode); + return io.out.map((plane) => plane.length); }, [ - planes.map((plane) => - Buffer.from(plane.buffer, plane.byteOffset, plane.length * 4).toString("base64"), - ), sampleRate, JSON.stringify(chain), options.automation ? serializeAutomation(options.automation) : "", - ] as [string[], number, string, string], - )) as string[]; + ] as [number, string, string], + )) as number[]; - // byteOffset and byteLength matter: Node pools small allocations, so a - // short payload decodes into an 8 KiB pool and a view over the whole - // ArrayBuffer would read kilobytes of unrelated memory at the wrong length. - const outPlanes = rendered.map((b64) => { - const buf = Buffer.from(b64, "base64"); - return new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); - }); + const outPlanes: Float32Array[] = []; + for (let p = 0; p < outLengths.length; p += 1) { + const byteLength = (outLengths[p] ?? 0) * 4; + const parts: Buffer[] = []; + for (let at = 0; at < byteLength; at += TRANSFER_BYTES) { + const b64 = (await page.evaluate( + ([index, offset, limit]: [number, number, number]) => { + const plane = (window as unknown as AudioFxWindow).__HF_FX_IO?.out[index]; + if (!plane) return ""; + const u8 = new Uint8Array( + plane.buffer, + plane.byteOffset + offset, + Math.min(limit, plane.length * 4 - offset), + ); + let s = ""; + const CHUNK = 0x8000; + for (let i = 0; i < u8.length; i += CHUNK) { + // `apply` takes array-likes, so the subarray goes in as it is; + // Array.from boxed every byte of a 32 KiB window for nothing. + s += String.fromCharCode.apply( + null, + u8.subarray(i, i + CHUNK) as unknown as number[], + ); + } + return btoa(s); + }, + [p, at, TRANSFER_BYTES] as [number, number, number], + )) as string; + parts.push(Buffer.from(b64, "base64")); + } + // byteOffset and byteLength matter: Node pools small allocations, so a + // short payload decodes into an 8 KiB pool and a view over the whole + // ArrayBuffer would read kilobytes of unrelated memory at the wrong length. + const buf = Buffer.concat(parts); + outPlanes.push( + new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)), + ); + } if (outPlanes.length === 0 || (outPlanes[0]?.length ?? 0) === 0) { throw new AudioFxRenderError(`Audio FX produced no samples for track ${options.trackId}`); } + // Null when the keyframes normalise away to nothing — then the mixer's + // own paths still own this track's gain, so say so rather than claiming + // a bake that never happened. + const gainAt = options.envelope + ? createEnvelopeWalker( + options.envelope.keyframes, + options.envelope.trackStart, + options.envelope.baseVolume, + ) + : null; + if (gainAt) applyEnvelopeToPlanes(outPlanes, sampleRate, gainAt); writeWav(outputWav, interleave(outPlanes), sampleRate, outPlanes.length); - return outputWav; + return { path: outputWav, envelopeBaked: gainAt !== null }; } finally { await page.close().catch(() => undefined); } diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index 828b76e8d4..cac9b10ae8 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -44,11 +44,15 @@ vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { // The FX render drives a headless browser; the mix only needs to know the // processed file exists and how long a tail the chain asked for. const { applyAudioFxChainMock } = vi.hoisted(() => ({ - applyAudioFxChainMock: vi.fn(async (_src: string, _chain: unknown, outPath: string) => { - const { writeFileSync } = await import("node:fs"); - writeFileSync(outPath, "stub"); - return outPath; - }), + applyAudioFxChainMock: vi.fn( + async (_src: string, _chain: unknown, outPath: string, options?: { envelope?: unknown }) => { + const { writeFileSync } = await import("node:fs"); + writeFileSync(outPath, "stub"); + // The real one bakes the volume envelope into its float output, so the + // mixer must not run its own pass afterwards. + return { path: outPath, envelopeBaked: Boolean(options?.envelope) }; + }, + ), })); vi.mock("./audioFxRender.js", async (importOriginal) => { @@ -295,6 +299,62 @@ describe("processCompositionAudio", () => { expect(filter).toContain("apad,atrim=0:8"); }); + it("hands the volume envelope to the FX pass instead of ducking the file after it", async () => { + // The FX pass writes 16-bit PCM, so a chain that overshoots full scale is + // clipped there. Ducking afterwards bakes that distortion in even though + // the lane pulls the track well down; the envelope has to travel into the + // FX pass and land on its float output. + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "voice.wav"), "stub"); + + const result = await processCompositionAudio( + [ + { + id: "voice", + src: "voice.wav", + start: 2, + end: 5, + mediaStart: 0, + layer: 0, + volume: 0.4, + volumeKeyframes: [ + { time: 2, volume: 1 }, + { time: 5, volume: 0.25 }, + ], + type: "audio", + fxChain: JSON.stringify({ + version: 1, + nodes: [{ type: "peaking", id: "p", params: { frequency: 440, gain: 12, q: 1 } }], + }), + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 5, + ); + + expect(result.success).toBe(true); + expect(applyAudioFxChainMock).toHaveBeenCalledTimes(1); + expect(applyAudioFxChainMock.mock.calls[0]?.[3]).toMatchObject({ + envelope: { + keyframes: [ + { time: 2, volume: 1 }, + { time: 5, volume: 0.25 }, + ], + trackStart: 2, + baseVolume: 0.4, + }, + }); + + // And the mixer trusts that bake: unity gain, no second pass, no ffmpeg + // volume expression re-applying the same envelope on top of it. + const filter = capturedFilterScripts[capturedFilterScripts.length - 1]; + expect(filter).not.toContain(":eval=frame"); + }); + it("cuts at the clip boundary when the chain has no tail", async () => { const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 10c6f9ad54..e71d26587b 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -962,6 +962,27 @@ export async function processCompositionAudio( ) : null; + // Computed before the chain runs, not after: the FX pass bakes the + // envelope into its float output so the duck lands before the ±1 clamp + // in writeWav, instead of after it. + // + // A volume lane supersedes keyframes probed from the timeline: the two + // would fight, and the lane is the explicit one. `lint` warns when a + // track carries both. + const laneKeyframes = automation + ? volumeLaneKeyframes(automation, element.start, element.end - element.start) + : null; + const envelopeKeyframes = laneKeyframes ?? element.volumeKeyframes; + const envelope = + envelopeKeyframes && envelopeKeyframes.length > 0 + ? { + keyframes: envelopeKeyframes, + trackStart: element.start, + baseVolume: element.volume ?? 1.0, + } + : null; + + let bakedEnvelope = false; let tailSeconds = 0; if (element.fxChain) { // The chain is serialised into the attribute, the same way colour @@ -971,7 +992,7 @@ export async function processCompositionAudio( // The rendered WAV is longer than the input by exactly this much, so // the mix has to be told to let it through. tailSeconds = chainTailSeconds(chain, automation ?? undefined); - audioSrcPath = await applyAudioFxChain( + const fxResult = await applyAudioFxChain( audioSrcPath, chain, join(workDir, `${element.id}-fx.wav`), @@ -979,29 +1000,24 @@ export async function processCompositionAudio( trackId: element.id, signal: effectiveSignal, ...(automation ? { automation } : {}), + ...(envelope ? { envelope } : {}), }, ); + audioSrcPath = fxResult.path; + bakedEnvelope = fxResult.envelopeBaked; } - // Primary volume-automation path: bake the envelope into the PCM samples - // (sample-accurate, no keyframe ceiling). If the WAV isn't the expected - // 16-bit PCM, fall back to the ffmpeg expression path by leaving the - // keyframes on the track for buildVolumeExpression to handle. - // - // A volume lane supersedes keyframes probed from the timeline: the two - // would fight, and the lane is the explicit one. `lint` warns when a - // track carries both. - const laneKeyframes = automation - ? volumeLaneKeyframes(automation, element.start, element.end - element.start) - : null; - const envelopeKeyframes = laneKeyframes ?? element.volumeKeyframes; - let bakedEnvelope = false; - if (envelopeKeyframes && envelopeKeyframes.length > 0) { + // Primary volume-automation path for a track the FX pass did not bake: + // multiply the envelope into the PCM samples (sample-accurate, no + // keyframe ceiling). If the WAV isn't the expected 16-bit PCM, fall + // back to the ffmpeg expression path by leaving the keyframes on the + // track for buildVolumeExpression to handle. + if (envelope && !bakedEnvelope) { bakedEnvelope = applyVolumeEnvelopeToWav( audioSrcPath, - envelopeKeyframes, - element.start, - element.volume ?? 1.0, + envelope.keyframes, + envelope.trackStart, + envelope.baseVolume, ); } tracks.push({ diff --git a/packages/engine/src/services/audioVolumeEnvelope.ts b/packages/engine/src/services/audioVolumeEnvelope.ts index 9d4d0aa730..08f8a18828 100644 --- a/packages/engine/src/services/audioVolumeEnvelope.ts +++ b/packages/engine/src/services/audioVolumeEnvelope.ts @@ -74,6 +74,40 @@ function parseWavLayout(buffer: Buffer): WavLayout | null { }; } +/** + * A gain lookup that walks forward through the envelope with a segment cursor, + * so a whole track costs O(N+M) rather than O(N×M). `interpolateVolumeGain` + * restarts from segment 0 on every call — fine for the preview path (once per + * RAF tick), not for a per-sample walk over 48k×duration frames. + * + * The cursor only ever advances, so callers must pass non-decreasing times. + * Returns null when the keyframes normalise to nothing, which the callers read + * as "no automation here". + */ +export function createEnvelopeWalker( + keyframes: AudioVolumeKeyframe[], + trackStart: number, + baseVolume: number, +): ((time: number) => number) | null { + const envelope = normaliseEnvelope(keyframes, trackStart, baseVolume); + const first = envelope[0]; + if (!first) return null; + + let segment = 0; + return (time: number): number => { + for (;;) { + const next = envelope[segment + 1]; + if (segment >= envelope.length - 2 || !next || time < next.time) break; + segment += 1; + } + const a = envelope[segment] ?? first; + const b = envelope[segment + 1] ?? a; + const span = b.time - a.time; + const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (time - a.time) / span)); + return a.volume + (b.volume - a.volume) * progress; + }; +} + /** * Multiply a prepared WAV's samples by a time-varying gain envelope in place. * @@ -86,8 +120,8 @@ export function applyVolumeEnvelopeToWav( trackStart: number, baseVolume: number, ): boolean { - const envelope = normaliseEnvelope(keyframes, trackStart, baseVolume); - if (envelope.length === 0) return false; + const gainAt = createEnvelopeWalker(keyframes, trackStart, baseVolume); + if (!gainAt) return false; try { const buffer = readFileSync(wavPath); @@ -99,21 +133,8 @@ export function applyVolumeEnvelopeToWav( const frameBytes = numChannels * bytesPerSample; const frameCount = Math.floor(dataSize / frameBytes); - // Maintain an incremental segment cursor so the per-frame envelope lookup - // is O(N+M) overall, not O(N×M). interpolateVolumeGain restarts from 0 on - // each call — fine for the preview path (one call per RAF tick) but not for - // the PCM path (one call per sample, 48k×duration frames total). - let segment = 0; for (let frame = 0; frame < frameCount; frame += 1) { - const time = frame / sampleRate; - while (segment < envelope.length - 2 && time >= envelope[segment + 1]!.time) segment += 1; - - const a = envelope[segment]!; - const b = envelope[segment + 1] ?? a; - const span = b.time - a.time; - const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (time - a.time) / span)); - const gain = a.volume + (b.volume - a.volume) * progress; - + const gain = gainAt(frame / sampleRate); const base = dataOffset + frame * frameBytes; for (let channel = 0; channel < numChannels; channel += 1) { const at = base + channel * bytesPerSample; diff --git a/plans/audio-automation-lanes/SPEC.md b/plans/audio-automation-lanes/SPEC.md index 85e32b67a4..f457d68f74 100644 --- a/plans/audio-automation-lanes/SPEC.md +++ b/plans/audio-automation-lanes/SPEC.md @@ -111,44 +111,6 @@ valid — they just can't be automation targets until the panel touches them. **Normalization** (`normalizeAutomation`, mirrors `normalizeAudioFxParams`): -- points sorted by `t`; duplicate `t` keeps the later point -- `v` clamped to the target's registry range; non-finite → point dropped -- lanes targeting a node id that no longer exists in the chain are **dropped** - (the author deleted the device; its automation dies with it — panel also - removes them eagerly on node delete) -- 1-point lane = constant; empty lanes array = attribute removed - -**Precedence for volume** (documented + linted): -`data-automation` volume lane → GSAP volume tween → `data-volume`. -New lint rule `audio_volume_double_automation` (warning) when an element has -both a volume lane and a GSAP tween on `volume`. - -## 5. Interpolation semantics - -- Between points: linear in the parameter's **working domain**. Params with - registry `scale: "log"` (frequency, some times) interpolate in log domain — - a 200 Hz → 8 kHz sweep is perceptually linear, matching what a DAW does. -- `curve` bends the segment: `f(x) = x^(2^(k·s))` shaping applied in the - working domain (s = curve, k ≈ 2). Exact constant chosen to visually match - Ableton's feel; pinned by unit tests once chosen. -- Before the first point: hold first value. After the last: hold last value. -- One shared implementation `sampleAutomationLane(lane, t)` in core — used by - the lane renderer (drawing), the scheduler (curve sampling), and the render - path. One interpolator, three consumers, or preview and picture drift. - -## 6. Preview architecture - -Scheduling hooks into `schedulePlayback` (transport), which already runs on -play / seek / rate change with the clip's `elapsed` offset: - -- **Volume lane** → scheduled on the source's existing `gainNode.gain` - (post-FX, i.e. fader semantics — matches Ableton, matches the render order - where FX runs before the volume bake). -- **FX param lanes** → scheduled on AudioParams exposed by the graph builders - (§8) of the chain instance spliced for this source. - -Mechanics per lane, at schedule time: - 1. Convert clip-local envelope → context-time segments starting at `scheduledAt`, offset by `elapsed`, scaled by playback rate. 2. Linear segments → `setValueAtTime` + `linearRampToValueAtTime` (log-domain