From 6a766b639984f0021c106ad37001bb72d7eb706b Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Aug 2026 21:49:39 +0000 Subject: [PATCH 1/3] feat(engine): stamp rendered files with hidden renderer provenance --- docs/guides/rendering.mdx | 29 +++ .../cli/src/background-removal/pipeline.ts | 4 +- packages/engine/src/index.ts | 10 + .../engine/src/services/chunkEncoder.test.ts | 6 + packages/engine/src/services/chunkEncoder.ts | 9 + .../engine/src/services/streamingEncoder.ts | 4 + packages/engine/src/utils/ffprobe.ts | 5 +- .../engine/src/utils/renderProvenance.test.ts | 220 ++++++++++++++++++ packages/engine/src/utils/renderProvenance.ts | 101 ++++++++ 9 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 packages/engine/src/utils/renderProvenance.test.ts create mode 100644 packages/engine/src/utils/renderProvenance.ts diff --git a/docs/guides/rendering.mdx b/docs/guides/rendering.mdx index 68716faaeb..a83df9e7f8 100644 --- a/docs/guides/rendering.mdx +++ b/docs/guides/rendering.mdx @@ -147,6 +147,35 @@ For several variable-driven versions, use batch rendering. For remote infrastruc Those workflows involve output naming, credentials, concurrency, and infrastructure choices. Start in the [CLI guide](/developers/cli) and use the complete [CLI reference](/packages/cli) when you need every flag. +## Render provenance + +Rendered video carries two container metadata tags that say which tool wrote the file: + +```bash +ffprobe -v error -show_entries format_tags -of json out.mp4 +``` + +```json +{ "hyperframes_renderer": "hyperframes", "hyperframes_version": "0.7.107" } +``` + +That is the whole of it. The tags name the renderer and its version, and nothing else: no file +paths, usernames, machine names, project names, or anything about the composition. They are +container metadata, not a visible watermark, so no pixel of your video changes. Matroska +uppercases tag names on read, so a `.webm` reports `HYPERFRAMES_RENDERER`. + +Strip them whenever you like: + +```bash +ffmpeg -i out.mp4 -map_metadata -1 -c copy clean.mp4 +``` + + + Treat the tags as a positive signal only. If they are present, HyperFrames wrote the file. If they + are absent that proves nothing, because re-encoding, remuxing, or any tool that drops unknown keys + removes them, and files rendered by older versions never carried them. + + ## If rendering fails Run: diff --git a/packages/cli/src/background-removal/pipeline.ts b/packages/cli/src/background-removal/pipeline.ts index ff19979263..c50233ee47 100644 --- a/packages/cli/src/background-removal/pipeline.ts +++ b/packages/cli/src/background-removal/pipeline.ts @@ -18,7 +18,7 @@ import { extname } from "node:path"; import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js"; import { createSession, type Session } from "./inference.js"; import { type Device, type ModelId } from "./manager.js"; -import { DEFAULT_VP9_CPU_USED } from "@hyperframes/engine"; +import { DEFAULT_VP9_CPU_USED, renderProvenanceArgs } from "@hyperframes/engine"; export type OutputFormat = "webm" | "mov" | "png"; @@ -182,6 +182,7 @@ export function buildEncoderArgs( "-metadata:s:v:0", "alpha_mode=1", "-an", + ...renderProvenanceArgs(outputPath), outputPath, ]; } @@ -197,6 +198,7 @@ export function buildEncoderArgs( "-pix_fmt", "yuva444p10le", "-an", + ...renderProvenanceArgs(outputPath), outputPath, ]; } diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index a10070ce1f..b8f8177f3b 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -393,3 +393,13 @@ export { type HdrMasteringMetadata, } from "./utils/hdr.js"; export type { VideoColorSpace } from "./utils/ffprobe.js"; +export { + renderProvenanceArgs, + appendRenderProvenanceArgs, + readRenderProvenance, + PROVENANCE_RENDERER_TAG, + PROVENANCE_VERSION_TAG, + PROVENANCE_RENDERER_NAME, + PROVENANCE_VERSION, + type RenderProvenance, +} from "./utils/renderProvenance.js"; diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts index 80491e966a..b9471d2746 100644 --- a/packages/engine/src/services/chunkEncoder.test.ts +++ b/packages/engine/src/services/chunkEncoder.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, it, expect, vi } from "vitest"; import { ENCODER_PRESETS, getEncoderPreset, buildEncoderArgs } from "./chunkEncoder.js"; +import { renderProvenanceArgs } from "../utils/renderProvenance.js"; const TINY_PNG = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAACXBIWXMAAAABAAAAAQBPJcTWAAAAEElEQVR4nGP8wwACLGCSAQANBAECv1AVswAAAABJRU5ErkJggg==", @@ -403,6 +404,7 @@ describe("muxVideoWithAudio audio codec handling", () => { "+faststart", "-avoid_negative_ts", "make_zero", + ...renderProvenanceArgs("/tmp/output.mp4"), "-r", "30", "-y", @@ -410,6 +412,10 @@ describe("muxVideoWithAudio audio codec handling", () => { ]); expect(calls[0]!.args).not.toContain("-shortest"); expect(calls[0]!.args).not.toContain("-use_editlist"); + // The faststart flag set above must survive the provenance flag: ffmpeg + // takes the last -movflags occurrence, and a non-additive one would drop it. + expect(calls[0]!.args.filter((a) => a === "-movflags")).toHaveLength(2); + expect(calls[0]!.args).toContain("+faststart"); emitClose(calls[0]!.proc, 0); await expect(muxPromise).resolves.toMatchObject({ diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts index 709d89ac49..95ede8161b 100644 --- a/packages/engine/src/services/chunkEncoder.ts +++ b/packages/engine/src/services/chunkEncoder.ts @@ -22,6 +22,7 @@ import { extractAudioMetadata } from "../utils/ffprobe.js"; import { type Fps, fpsToFfmpegArg } from "@hyperframes/core"; import type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js"; import { appendVp9CpuUsedArg } from "./vp9Options.js"; +import { appendRenderProvenanceArgs } from "../utils/renderProvenance.js"; export type { EncoderOptions, EncodeResult, MuxResult } from "./chunkEncoder.types.js"; @@ -361,6 +362,7 @@ export function buildEncoderArgs( } else if (codec === "prores") { args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0"); args.push("-pix_fmt", pixelFormat); + appendRenderProvenanceArgs(args, outputPath); return [...args, "-y", outputPath]; } @@ -445,6 +447,8 @@ export function buildEncoderArgs( args.push("-avoid_negative_ts", "make_zero"); + appendRenderProvenanceArgs(args, outputPath); + args.push("-y", outputPath); return args; } @@ -701,6 +705,10 @@ export async function muxVideoWithAudio( // AAC priming packet. `make_zero` discards that edit and shifts copied video // forward by one AAC frame (~21ms), creating a visible first-frame offset. if (!copiesContainerizedAac) args.push("-avoid_negative_ts", "make_zero"); + // Re-assert provenance here: this stage re-muxes into the delivered + // container, and the mp4 muxer drops the encode stage's tags without the + // use_metadata_tags flag that appendRenderProvenanceArgs adds. + appendRenderProvenanceArgs(args, outputPath); if (fps !== undefined) { // Set the exact output framerate so the muxer doesn't PTS-average a // fractional rational like `360000/12001` instead of `30/1` into the @@ -742,6 +750,7 @@ export async function applyFaststart( return { success: true, outputPath, durationMs: 0 }; } const args = ["-i", inputPath, "-c", "copy", "-movflags", "+faststart"]; + appendRenderProvenanceArgs(args, outputPath); if (fps !== undefined) { // Set the exact output framerate so the final remux doesn't PTS-average // a fractional rational like `360000/12001` instead of `30/1` into the diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.ts index 7e896f9853..545c56024f 100644 --- a/packages/engine/src/services/streamingEncoder.ts +++ b/packages/engine/src/services/streamingEncoder.ts @@ -36,6 +36,7 @@ import { withEvenDimensionPad } from "../utils/evenDimensions.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { fpsToFfmpegArg, type Fps } from "@hyperframes/core"; import { appendVp9CpuUsedArg } from "./vp9Options.js"; +import { appendRenderProvenanceArgs } from "../utils/renderProvenance.js"; // Re-export EncoderOptions so callers can reference the type via this module. export type { EncoderOptions } from "./chunkEncoder.types.js"; @@ -350,6 +351,7 @@ export function buildStreamingArgs( } else if (codec === "prores") { args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0"); args.push("-pix_fmt", pixelFormat); + appendRenderProvenanceArgs(args, outputPath); return [...args, "-y", outputPath]; } @@ -428,6 +430,8 @@ export function buildStreamingArgs( // for the full explanation; same playback compatibility class. args.push("-avoid_negative_ts", "make_zero"); + appendRenderProvenanceArgs(args, outputPath); + args.push("-y", outputPath); return args; } diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts index 006d6ef084..f4f6ccc3f9 100644 --- a/packages/engine/src/utils/ffprobe.ts +++ b/packages/engine/src/utils/ffprobe.ts @@ -608,7 +608,10 @@ function extractStillImageMetadata(filePath: string): StillImageMetadata | null * in newer ones; HDR tags vary similarly. Use this for any sidecar tag where * you want to be resilient across muxer versions. */ -function readTagCI(tags: Record | undefined, name: string): string { +export function readTagCI( + tags: Record | undefined, + name: string, +): string { if (!tags) return ""; const target = name.toLowerCase(); for (const [key, value] of Object.entries(tags)) { diff --git a/packages/engine/src/utils/renderProvenance.test.ts b/packages/engine/src/utils/renderProvenance.test.ts new file mode 100644 index 0000000000..b6ab67c005 --- /dev/null +++ b/packages/engine/src/utils/renderProvenance.test.ts @@ -0,0 +1,220 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { getFfmpegBinary, getFfprobeBinary } from "./ffmpegBinaries.js"; +import { + PROVENANCE_RENDERER_NAME, + PROVENANCE_RENDERER_TAG, + PROVENANCE_VERSION, + PROVENANCE_VERSION_TAG, + readRenderProvenance, + renderProvenanceArgs, +} from "./renderProvenance.js"; +import { applyFaststart, buildEncoderArgs, muxVideoWithAudio } from "../services/chunkEncoder.js"; + +/** Same probe the ffmpeg-dependent suites use: ask the binary, don't assume it. */ +const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0; + +describe("renderProvenanceArgs", () => { + it("tags the renderer and version", () => { + const args = renderProvenanceArgs("out.mp4"); + expect(args).toContain(`${PROVENANCE_RENDERER_TAG}=${PROVENANCE_RENDERER_NAME}`); + expect(args).toContain(`${PROVENANCE_VERSION_TAG}=${PROVENANCE_VERSION}`); + }); + + it("carries nothing that identifies the machine or the project", () => { + // Provenance is renderer identity only. A path, username or composition + // name here would travel with every distributed file. + const values = renderProvenanceArgs("/home/someone/projects/secret-launch/out.mp4") + .filter((a) => a.includes("=")) + .join(" "); + expect(values).not.toMatch(/secret-launch|someone|\/home\//); + }); + + it("adds use_metadata_tags for the mov-family containers only", () => { + for (const ext of ["mp4", "mov", "m4v"]) { + expect(renderProvenanceArgs(`out.${ext}`)).toContain("-movflags"); + } + expect(renderProvenanceArgs("out.webm")).not.toContain("-movflags"); + }); + + it("matches the container case-insensitively", () => { + expect(renderProvenanceArgs("OUT.MP4")).toContain("-movflags"); + }); + + it("uses the additive + form so it cannot clobber an earlier -movflags", () => { + // Regression guard. A bare `use_metadata_tags` resets the flag field and + // silently drops `+faststart` set earlier in the same command: tags still + // probe correctly, but the moov atom moves to the end of the file. The + // real-encode case below proves the behaviour; this pins the argument. + const args = renderProvenanceArgs("out.mp4"); + expect(args[args.indexOf("-movflags") + 1]).toBe("+use_metadata_tags"); + }); +}); + +describe("readRenderProvenance", () => { + it("reads mp4-cased tags", () => { + expect( + readRenderProvenance({ hyperframes_renderer: "hyperframes", hyperframes_version: "1.2.3" }), + ).toEqual({ renderer: "hyperframes", version: "1.2.3" }); + }); + + it("reads matroska-uppercased tags", () => { + // Matroska uppercases keys on read; a case-sensitive lookup would work on + // mp4 and miss every webm. + expect( + readRenderProvenance({ HYPERFRAMES_RENDERER: "hyperframes", HYPERFRAMES_VERSION: "1.2.3" }), + ).toEqual({ renderer: "hyperframes", version: "1.2.3" }); + }); + + it("returns null when there is no provenance", () => { + expect(readRenderProvenance({ major_brand: "isom" })).toBeNull(); + expect(readRenderProvenance(undefined)).toBeNull(); + }); +}); + +describe.skipIf(!HAS_FFMPEG)("provenance survives a real encode", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "hf-provenance-")); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + const run = (args: string[]): void => { + const res = spawnSync(getFfmpegBinary(), args, { encoding: "utf-8" }); + if (res.status !== 0) throw new Error(`ffmpeg failed: ${res.stderr ?? ""}`); + }; + + /** Format tags as ffprobe reports them, which is the only thing that counts. */ + const formatTags = (file: string): Record => { + const res = spawnSync( + getFfprobeBinary(), + ["-v", "error", "-show_entries", "format_tags", "-of", "json", "--", file], + { encoding: "utf-8" }, + ); + if (res.status !== 0) throw new Error(`ffprobe failed: ${res.stderr ?? ""}`); + const parsed = JSON.parse(res.stdout) as { format?: { tags?: Record } }; + return parsed.format?.tags ?? {}; + }; + + const source = (): string => { + const src = join(dir, "src.mp4"); + run([ + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc=size=160x120:rate=15:duration=1", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-y", + src, + ]); + return src; + }; + + // The whole point of the feature: assert against the encoded file, not the + // argument array. mp4 accepts `-metadata` for an unknown key and then + // silently discards it, so an args-only test passes while every shipped mp4 + // carries no provenance at all. + it.each(["mp4", "mov", "webm"])("round-trips through a real %s encode", (ext) => { + const out = join(dir, `out.${ext}`); + const codec = ext === "webm" ? ["-c:v", "libvpx-vp9", "-b:v", "200k"] : ["-c:v", "libx264"]; + run([ + "-v", + "error", + "-i", + source(), + ...codec, + "-pix_fmt", + "yuv420p", + ...renderProvenanceArgs(out), + "-y", + out, + ]); + + expect(readRenderProvenance(formatTags(out))).toEqual({ + renderer: PROVENANCE_RENDERER_NAME, + version: PROVENANCE_VERSION, + }); + }); + + it("keeps +faststart working alongside the provenance flag", () => { + // Guards the clobber directly: with a non-additive `use_metadata_tags` the + // tags below still pass while the moov atom silently moves to the end. + const out = join(dir, "faststart.mp4"); + run([ + "-v", + "error", + "-i", + source(), + "-c", + "copy", + "-movflags", + "+faststart", + ...renderProvenanceArgs(out), + "-y", + out, + ]); + + expect(readRenderProvenance(formatTags(out))).not.toBeNull(); + + // faststart means the moov atom precedes mdat. Read the whole file and + // compare offsets: both atoms are always present, so a missing one would + // mean the file is malformed rather than merely unoptimised. + const bytes = readFileSync(out); + const moov = bytes.indexOf("moov", 0, "latin1"); + const mdat = bytes.indexOf("mdat", 0, "latin1"); + expect(moov).toBeGreaterThan(-1); + expect(mdat).toBeGreaterThan(-1); + expect(moov).toBeLessThan(mdat); + }); + + it("survives the encode -> mux -> faststart chain that produces a delivered mp4", async () => { + // The stage that actually bites: `muxVideoWithAudio` and `applyFaststart` + // each run their own ffmpeg over the encoder's output. Tagging only at the + // encode stage passes an args test and still ships an mp4 with no + // provenance, because the mux drops unknown keys on the way through. + const encoded = join(dir, "encoded.mp4"); + run([ + "-v", + "error", + "-i", + source(), + ...buildEncoderArgs({ fps: { num: 15, den: 1 }, width: 160, height: 120 }, [], encoded), + ]); + expect(readRenderProvenance(formatTags(encoded))).not.toBeNull(); + + const audio = join(dir, "audio.m4a"); + run([ + "-v", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=1", + "-c:a", + "aac", + "-y", + audio, + ]); + + const muxed = join(dir, "muxed.mp4"); + expect((await muxVideoWithAudio(encoded, audio, muxed)).success).toBe(true); + expect(readRenderProvenance(formatTags(muxed))).not.toBeNull(); + + const delivered = join(dir, "delivered.mp4"); + expect((await applyFaststart(muxed, delivered)).success).toBe(true); + expect(readRenderProvenance(formatTags(delivered))).toEqual({ + renderer: PROVENANCE_RENDERER_NAME, + version: PROVENANCE_VERSION, + }); + }); +}); diff --git a/packages/engine/src/utils/renderProvenance.ts b/packages/engine/src/utils/renderProvenance.ts new file mode 100644 index 0000000000..cb38f5d2c7 --- /dev/null +++ b/packages/engine/src/utils/renderProvenance.ts @@ -0,0 +1,101 @@ +import { createRequire } from "node:module"; +import { readTagCI } from "./ffprobe.js"; + +/** + * Hidden render provenance. + * + * HyperFrames stamps the *container* — never the picture — with the renderer + * name and version, so a rendered file can be attributed later without a + * visible watermark burned into the frames. + * + * What goes in is deliberately boring: renderer name and version. No file + * paths, usernames, machine names, project names or composition content. Once + * a file is distributed the metadata travels with it, and metadata leaks are + * hard to walk back. + * + * **Positive signal only.** The tags being present means HyperFrames wrote the + * file. Their absence proves nothing: re-encoding, remuxing, or any tool that + * drops unknown keys removes them, and files rendered before this feature + * never had them. Do not build "no tag therefore not HyperFrames" logic on it. + */ + +export const PROVENANCE_RENDERER_TAG = "hyperframes_renderer"; +export const PROVENANCE_VERSION_TAG = "hyperframes_version"; +export const PROVENANCE_RENDERER_NAME = "hyperframes"; + +const UNKNOWN_VERSION = "0.0.0-dev"; + +function readEngineVersion(): string { + try { + // The engine ships as raw TS and exports "./package.json", so this + // resolves without a build-time define. A failed read must never break a + // render, hence the fallback rather than a throw. + const version = (createRequire(import.meta.url)("../../package.json") as { version?: string }) + .version; + return typeof version === "string" && version.length > 0 ? version : UNKNOWN_VERSION; + } catch { + return UNKNOWN_VERSION; + } +} + +export const PROVENANCE_VERSION = readEngineVersion(); + +/** + * MP4/MOV (the mov muxer family) writes only tags it recognises from a fixed + * map and silently discards everything else. WebM/Matroska keeps arbitrary + * keys as-is. + */ +function isMovFamilyContainer(outputPath: string): boolean { + const lower = outputPath.toLowerCase(); + return lower.endsWith(".mp4") || lower.endsWith(".mov") || lower.endsWith(".m4v"); +} + +/** + * The provenance ffmpeg arguments for a given output container. Place them + * before the output path. + * + * Must be applied on *every* stage that writes an mp4 — encode, mux and the + * faststart remux each run their own ffmpeg, and a stage without the flag + * drops the tags written by the stage before it. + */ +export function renderProvenanceArgs(outputPath: string): string[] { + const args = [ + "-metadata", + `${PROVENANCE_RENDERER_TAG}=${PROVENANCE_RENDERER_NAME}`, + "-metadata", + `${PROVENANCE_VERSION_TAG}=${PROVENANCE_VERSION}`, + ]; + + if (isMovFamilyContainer(outputPath)) { + // The additive `+` form is mandatory. A bare `-movflags use_metadata_tags` + // *resets* the flag field, silently discarding a `+faststart` set earlier + // in the same command — the file still probes fine and keeps its tags, + // but the moov atom lands at the end and progressive playback regresses. + args.push("-movflags", "+use_metadata_tags"); + } + return args; +} + +/** Mutating form of {@link renderProvenanceArgs} for push-built arg lists. */ +export function appendRenderProvenanceArgs(args: string[], outputPath: string): void { + args.push(...renderProvenanceArgs(outputPath)); +} + +export interface RenderProvenance { + renderer: string; + /** Empty string when the renderer tag is present but the version is not. */ + version: string; +} + +/** + * Read provenance back from ffprobe format tags. Matroska uppercases keys on + * read while mp4 preserves the case written, so the lookup is + * case-insensitive — a case-sensitive read works on mp4 and misses every webm. + */ +export function readRenderProvenance( + tags: Record | undefined, +): RenderProvenance | null { + const renderer = readTagCI(tags, PROVENANCE_RENDERER_TAG); + if (renderer === "") return null; + return { renderer, version: readTagCI(tags, PROVENANCE_VERSION_TAG) }; +} From 07aa07af1521b971c6a18790e3a2825abc839e7f Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Aug 2026 22:58:21 +0000 Subject: [PATCH 2/3] fix(engine,producer): re-assert provenance at every container writer Review found that a no-audio MOV render still shipped untagged. The concat step is the last container write on that path (mux is skipped without audio, and applyFaststart only copies mov/webm), and the concat demuxer does not carry the chunks' container metadata through. The same hole applies to no-audio WebM, and to the in-process chunked encode in chunkEncoder, not just the distributed assemble path. mp4 was masked throughout because applyFaststart re-runs ffmpeg for that format and re-tagged the output. Tags the four remaining writers: the chunked-encode concat, and assemble's single-chunk remux, concat and cfr re-encode. Also corrects the trust claim. These are unsigned, freely writable keys, so a present tag means the file claims to be HyperFrames output, not that HyperFrames wrote it. Documented as an unauthenticated diagnostic hint rather than an authenticity or attribution boundary. Tests assert on the assembled file through the real assemble() path for both mov and webm; both fail without the concat fix. --- docs/guides/rendering.mdx | 11 +- packages/engine/src/services/chunkEncoder.ts | 20 ++-- packages/engine/src/utils/renderProvenance.ts | 18 ++-- .../src/services/distributed/assemble.test.ts | 102 +++++++++++++++++- .../src/services/distributed/assemble.ts | 13 ++- 5 files changed, 137 insertions(+), 27 deletions(-) diff --git a/docs/guides/rendering.mdx b/docs/guides/rendering.mdx index a83df9e7f8..66da8cc742 100644 --- a/docs/guides/rendering.mdx +++ b/docs/guides/rendering.mdx @@ -171,9 +171,14 @@ ffmpeg -i out.mp4 -map_metadata -1 -c copy clean.mp4 ``` - Treat the tags as a positive signal only. If they are present, HyperFrames wrote the file. If they - are absent that proves nothing, because re-encoding, remuxing, or any tool that drops unknown keys - removes them, and files rendered by older versions never carried them. + These tags are an unauthenticated diagnostic hint, not proof of origin. They are ordinary unsigned + container keys, so anything can write the same two values with a single `ffmpeg -metadata` + command: a tag that is present means the file *claims* to be HyperFrames output, not that + HyperFrames wrote it. A tag that is absent means just as little, because re-encoding, remuxing, or + any tool that drops unknown keys strips it, and files rendered by older versions never carried it. + Treat it as a "what probably produced this file?" hint for support and debugging, never as an + authenticity, attribution, or licensing check. Verifiable provenance needs signed claims such as + C2PA. ## If rendering fails diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts index 95ede8161b..dbad9fc716 100644 --- a/packages/engine/src/services/chunkEncoder.ts +++ b/packages/engine/src/services/chunkEncoder.ts @@ -615,18 +615,14 @@ export async function encodeFramesChunkedConcat( const concatInput = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n"); writeFileSync(concatListPath, concatInput, "utf-8"); - const concatArgs = [ - "-f", - "concat", - "-safe", - "0", - "-i", - concatListPath, - "-c", - "copy", - "-y", - outputPath, - ]; + const concatArgs = ["-f", "concat", "-safe", "0", "-i", concatListPath, "-c", "copy"]; + // The concat demuxer does not carry per-chunk container metadata into the + // output, so the chunks' provenance is dropped here even though every chunk + // carries it. Re-assert on the concatenated file: for a no-audio mov/webm + // this is the last container write, since mux is skipped and applyFaststart + // only copies those two formats. + appendRenderProvenanceArgs(concatArgs, outputPath); + concatArgs.push("-y", outputPath); const encodeTimeout = config?.ffmpegEncodeTimeout ?? DEFAULT_CONFIG.ffmpegEncodeTimeout; const concatProcessResult = await runFfmpeg(concatArgs, { signal, timeout: encodeTimeout }); const concatResult = { diff --git a/packages/engine/src/utils/renderProvenance.ts b/packages/engine/src/utils/renderProvenance.ts index cb38f5d2c7..63087137c0 100644 --- a/packages/engine/src/utils/renderProvenance.ts +++ b/packages/engine/src/utils/renderProvenance.ts @@ -5,18 +5,24 @@ import { readTagCI } from "./ffprobe.js"; * Hidden render provenance. * * HyperFrames stamps the *container* — never the picture — with the renderer - * name and version, so a rendered file can be attributed later without a - * visible watermark burned into the frames. + * name and version, so a rendered file carries a machine-readable note about + * what produced it, with no visible watermark burned into the frames. * * What goes in is deliberately boring: renderer name and version. No file * paths, usernames, machine names, project names or composition content. Once * a file is distributed the metadata travels with it, and metadata leaks are * hard to walk back. * - * **Positive signal only.** The tags being present means HyperFrames wrote the - * file. Their absence proves nothing: re-encoding, remuxing, or any tool that - * drops unknown keys removes them, and files rendered before this feature - * never had them. Do not build "no tag therefore not HyperFrames" logic on it. + * **An unauthenticated hint — not an authenticity or attribution boundary.** + * These are ordinary unsigned container keys that any tool can write, so a + * present tag means the file *claims* to be HyperFrames output, not that + * HyperFrames produced it: one `ffmpeg -metadata hyperframes_renderer=...` + * forges it. Absence proves just as little, since re-encoding, remuxing, or + * any tool that drops unknown keys strips them, and files rendered before this + * feature never had them. Good for diagnostics and support ("what wrote this + * file?"); never a basis for trust, attribution or licensing decisions in + * either direction. Verifiable provenance needs signed claims (C2PA), which + * this deliberately is not. */ export const PROVENANCE_RENDERER_TAG = "hyperframes_renderer"; diff --git a/packages/producer/src/services/distributed/assemble.test.ts b/packages/producer/src/services/distributed/assemble.test.ts index 6a6e69c028..5eaa5aaa38 100644 --- a/packages/producer/src/services/distributed/assemble.test.ts +++ b/packages/producer/src/services/distributed/assemble.test.ts @@ -19,8 +19,15 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { + PROVENANCE_RENDERER_NAME, + PROVENANCE_VERSION, + readRenderProvenance, + renderProvenanceArgs, +} from "@hyperframes/engine"; import type { ChunkSliceJson } from "../render/stages/freezePlan.js"; import { assemble } from "./assemble.js"; +import type { DistributedFormat } from "./shared.js"; let runRoot: string; let hasFfmpeg = false; @@ -42,7 +49,7 @@ afterAll(() => { * loop. */ function buildPlanDir( - format: "mp4" | "png-sequence", + format: DistributedFormat, chunks: ChunkSliceJson[], totalFrames: number, hasAudio: boolean, @@ -110,6 +117,57 @@ function makeMp4Chunk(outputPath: string, frameCount: number): void { } } +/** + * Encode a tiny provenance-tagged chunk in `format`, mirroring what the chunk + * encoder writes. mov uses libx264 rather than production's ProRes: container + * metadata handling belongs to the muxer, not the codec, and h264-in-mov keeps + * the test fast and portable across CI ffmpeg builds. + */ +function makeTaggedChunk(outputPath: string, frameCount: number, format: "mov" | "webm"): void { + const codec = + format === "webm" + ? ["-c:v", "libvpx-vp9", "-b:v", "200k"] + : ["-c:v", "libx264", "-preset", "ultrafast"]; + const args = [ + "-v", + "error", + "-f", + "lavfi", + "-i", + `testsrc=size=160x120:rate=30:duration=${frameCount / 30}`, + ...codec, + "-g", + String(frameCount), + "-keyint_min", + String(frameCount), + "-pix_fmt", + "yuv420p", + "-vframes", + String(frameCount), + ...renderProvenanceArgs(outputPath), + "-y", + outputPath, + ]; + const result = spawnSync("ffmpeg", args, { stdio: "pipe" }); + if (result.status !== 0) { + throw new Error(`ffmpeg ${format} chunk failed: ${result.stderr.toString().slice(-400)}`); + } +} + +/** Read the provenance tags ffprobe actually reports for `outputPath`. */ +function probeProvenance(outputPath: string): { renderer: string; version: string } | null { + const result = spawnSync( + "ffprobe", + ["-v", "error", "-show_entries", "format_tags", "-of", "json", "--", outputPath], + { stdio: "pipe" }, + ); + if (result.status !== 0) return null; + const parsed = JSON.parse(result.stdout.toString()) as { + format?: { tags?: Record }; + }; + return readRenderProvenance(parsed.format?.tags ?? {}); +} + /** Generate an AAC audio file of `durationSeconds` of silence. */ function makeAacAudio(outputPath: string, durationSeconds: number): void { const result = spawnSync("ffmpeg", [ @@ -606,6 +664,48 @@ describe("assemble()", () => { TIMEOUT_MS, ); + // Regression: a distributed render with NO audio skips the mux entirely, and + // applyFaststart only copies mov/webm rather than re-running ffmpeg. That + // leaves the concat step as the last container write, and the concat demuxer + // does not carry the chunks' container metadata through — so before the + // provenance args were added here, both formats shipped with no tags at all + // while mp4 was silently rescued by faststart's re-mux. Asserting on the + // assembled file rather than the argv is the point: ffmpeg accepts the + // metadata flags either way and simply drops the keys. + it.each(["mov", "webm"] as const)( + "keeps render provenance on a no-audio %s render", + async (format) => { + if (!hasFfmpeg) { + console.warn(`[assemble.test] skipping ${format} provenance test — ffmpeg not available`); + return; + } + + const chunks: ChunkSliceJson[] = [ + { index: 0, startFrame: 0, endFrame: 5 }, + { index: 1, startFrame: 5, endFrame: 10 }, + ]; + const planDir = buildPlanDir(format, chunks, 10, false); + + const chunkAPath = join(planDir, `chunk-0.${format}`); + const chunkBPath = join(planDir, `chunk-1.${format}`); + makeTaggedChunk(chunkAPath, 5, format); + makeTaggedChunk(chunkBPath, 5, format); + // The chunks really are tagged, so a failure below is the assemble step + // dropping them rather than the fixture never having had them. + expect(probeProvenance(chunkAPath)).not.toBeNull(); + + const outputPath = join(planDir, `output.${format}`); + const result = await assemble(planDir, [chunkAPath, chunkBPath], null, outputPath); + + expect(existsSync(result.outputPath)).toBe(true); + expect(probeProvenance(outputPath)).toEqual({ + renderer: PROVENANCE_RENDERER_NAME, + version: PROVENANCE_VERSION, + }); + }, + TIMEOUT_MS, + ); + it("rejects when chunkPaths.length does not match chunks.json length", async () => { const chunks: ChunkSliceJson[] = [ { index: 0, startFrame: 0, endFrame: 5 }, diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.ts index e7478c41aa..309c26db88 100644 --- a/packages/producer/src/services/distributed/assemble.ts +++ b/packages/producer/src/services/distributed/assemble.ts @@ -35,6 +35,7 @@ import { } from "node:fs"; import { dirname, join } from "node:path"; import { + appendRenderProvenanceArgs, applyFaststart, MIXED_AUDIO_FILENAME, muxVideoWithAudio, @@ -177,7 +178,9 @@ export async function assemble( // touching the encoded stream. Multi-chunk renders continue through // the concat demuxer where the existing `-r` input flag works. if (chunkPaths.length === 1) { - const remuxArgs = ["-i", chunkPaths[0]!, "-c", "copy", "-r", fpsArg, "-y", concatOutputPath]; + const remuxArgs = ["-i", chunkPaths[0]!, "-c", "copy", "-r", fpsArg]; + appendRenderProvenanceArgs(remuxArgs, concatOutputPath); + remuxArgs.push("-y", concatOutputPath); const remuxResult = await runFfmpeg(remuxArgs, { signal: abortSignal }); if (!remuxResult.success) { throw new Error( @@ -210,9 +213,9 @@ export async function assemble( concatListPath, "-c", "copy", - "-y", - concatOutputPath, ]; + appendRenderProvenanceArgs(concatArgs, concatOutputPath); + concatArgs.push("-y", concatOutputPath); const concatResult = await runFfmpeg(concatArgs, { signal: abortSignal }); if (!concatResult.success) { throw new Error( @@ -280,9 +283,9 @@ export async function assemble( "cfr", "-r", fpsArg, - "-y", - cfrOutputPath, ]; + appendRenderProvenanceArgs(cfrArgs, cfrOutputPath); + cfrArgs.push("-y", cfrOutputPath); const cfrResult = await runFfmpeg(cfrArgs, { signal: abortSignal }); if (!cfrResult.success) { throw new Error( From d78655d5b656c5bf8d8e9acec3ab9f2aeb18d13b Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Aug 2026 23:02:32 +0000 Subject: [PATCH 3/3] test(engine): pin provenance through the in-process chunked concat Review noted the distributed writers are mutation-pinned but the encodeFramesChunkedConcat fix had no real-file regression of its own. Encodes 70 frames at a 30-frame chunk size so the concat step actually runs, then asserts the tags on the resulting no-audio mov. Fails without the concat fix, passes with it. --- .../engine/src/utils/renderProvenance.test.ts | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/packages/engine/src/utils/renderProvenance.test.ts b/packages/engine/src/utils/renderProvenance.test.ts index b6ab67c005..7c5a75b0a0 100644 --- a/packages/engine/src/utils/renderProvenance.test.ts +++ b/packages/engine/src/utils/renderProvenance.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -12,7 +12,12 @@ import { readRenderProvenance, renderProvenanceArgs, } from "./renderProvenance.js"; -import { applyFaststart, buildEncoderArgs, muxVideoWithAudio } from "../services/chunkEncoder.js"; +import { + applyFaststart, + buildEncoderArgs, + encodeFramesChunkedConcat, + muxVideoWithAudio, +} from "../services/chunkEncoder.js"; /** Same probe the ffmpeg-dependent suites use: ask the binary, don't assume it. */ const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0; @@ -177,6 +182,48 @@ describe.skipIf(!HAS_FFMPEG)("provenance survives a real encode", () => { expect(moov).toBeLessThan(mdat); }); + // The in-process sibling of the distributed-assemble regression. This path + // does its own concat-copy straight to the deliverable, and the concat + // demuxer does not carry the chunks' container metadata through — so for a + // no-audio mov (mux skipped, faststart only copies mov) the concat is the + // last container write and the tags have to be re-asserted there. mp4 would + // hide this: faststart re-muxes it and puts them back. + it("survives the in-process chunked-encode concat for a no-audio mov", async () => { + const framesDir = join(dir, "frames"); + mkdirSync(framesDir, { recursive: true }); + // 70 frames against a 30-frame chunk size gives 3 chunks, so the concat + // step actually runs. A single chunk would skip it entirely. + run([ + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc=size=160x120:rate=30:duration=2.34", + "-frames:v", + "70", + "-start_number", + "0", + "-y", + join(framesDir, "frame_%06d.png"), + ]); + + const out = join(dir, "chunked.mov"); + const result = await encodeFramesChunkedConcat( + framesDir, + "frame_%06d.png", + out, + { fps: { num: 30, den: 1 }, width: 160, height: 120, preset: "ultrafast" }, + 30, + ); + + expect(result.success).toBe(true); + expect(readRenderProvenance(formatTags(out))).toEqual({ + renderer: PROVENANCE_RENDERER_NAME, + version: PROVENANCE_VERSION, + }); + }, 60_000); + it("survives the encode -> mux -> faststart chain that produces a delivered mp4", async () => { // The stage that actually bites: `muxVideoWithAudio` and `applyFaststart` // each run their own ffmpeg over the encoder's output. Tagging only at the