diff --git a/packages/access/src/gate/errors.ts b/packages/access/src/gate/errors.ts index 84374d7..b0042c3 100644 --- a/packages/access/src/gate/errors.ts +++ b/packages/access/src/gate/errors.ts @@ -7,4 +7,4 @@ export function formatDenial(filePath: string, reasons: string[]): string { const bullets = reasons.map((r) => ` - ${r}`).join("\n"); return `open-wiki refused the write to ${filePath}:\n${bullets}`; -} \ No newline at end of file +} diff --git a/packages/access/src/gate/guard.ts b/packages/access/src/gate/guard.ts index 7877819..2b9ce9d 100644 --- a/packages/access/src/gate/guard.ts +++ b/packages/access/src/gate/guard.ts @@ -33,4 +33,4 @@ export function isConfigWrite(filePath: string, projectRoot: string): boolean { /** The reason the hook gives the agent when it refuses a config write (9.6/9.13). */ export function configWriteReason(filePath: string): string { return `open-wiki refuses writes to its own configuration (${filePath}). The gate, .mcp.json and CLAUDE.md are not editable through the agent.`; -} \ No newline at end of file +} diff --git a/packages/access/src/index.ts b/packages/access/src/index.ts index 20abed9..6fedf15 100644 --- a/packages/access/src/index.ts +++ b/packages/access/src/index.ts @@ -87,6 +87,8 @@ export { type SourceState, type SourceStage, } from "./sources/state.js"; +export { projectVocabulary, rankNames, DEFAULT_VOCABULARY_LIMIT } from "./sources/vocabulary.js"; +export { transcriptionInputs, type TranscriptionInputs } from "./sources/transcription.js"; export { uploadTextSource, writeSourceText, normaliseText } from "./sources/ingest.js"; export { uploadPdfSource, diff --git a/packages/access/src/sources/state.ts b/packages/access/src/sources/state.ts index 1406adf..d3a8fad 100644 --- a/packages/access/src/sources/state.ts +++ b/packages/access/src/sources/state.ts @@ -95,8 +95,23 @@ export function sourceState( citedBy: [...citedBy], }; - if (failed !== undefined && !textReady) { - return { ...base, stage: "failed", error: failed, ...progressOf(chunks.length, done) }; + // A chunk that failed is not a transcription that stopped. The pipeline + // (4.9) records a chunk's error and carries on to the next one, because 6.3 + // offers "redo only what failed" and that needs the rest attempted — so a + // single 429 twelve minutes into a healthy run would otherwise make the + // source read as `failed`, with a progress count that keeps climbing, for + // the rest of the run. + // + // `failed` means nothing is left to try: every chunk has been attempted and + // at least one did not succeed. The error is carried either way, so a caller + // showing a source in flight can still say what went wrong on the way. + const untried = chunks.some((c) => !c.done && c.error === undefined); + const stopped = + journal?.error !== undefined || (chunks.length > 0 && !untried && done < chunks.length); + const error = failed !== undefined ? { error: failed } : {}; + + if (stopped && !textReady) { + return { ...base, stage: "failed", ...error, ...progressOf(chunks.length, done) }; } // `textReady` gates "cited" on purpose: a page citing a source whose text // never landed is citing something nothing could have read, and reporting @@ -104,7 +119,7 @@ export function sourceState( // recorded in `citedBy`, so the caller can say both things. if (textReady) return { ...base, stage: citedBy.length > 0 ? "cited" : "text-ready" }; if (chunks.length > 0) { - return { ...base, stage: "transcribing", ...progressOf(chunks.length, done) }; + return { ...base, stage: "transcribing", ...error, ...progressOf(chunks.length, done) }; } return { ...base, stage: "received" }; } diff --git a/packages/access/src/sources/transcription.ts b/packages/access/src/sources/transcription.ts new file mode 100644 index 0000000..b5f6290 --- /dev/null +++ b/packages/access/src/sources/transcription.ts @@ -0,0 +1,35 @@ +import { readSettings, type Language } from "../config/settings.js"; +import { projectVocabulary, DEFAULT_VOCABULARY_LIMIT } from "./vocabulary.js"; + +/** + * What a transcription needs from the project, minus the credential (plan + * 4.10 and 4.15). + * + * **The credential is deliberately not here.** `config/secrets.ts` says it in + * as many words: the CLI, the hooks and the MCP process must not read the + * secret, because their stderr is consumed by an agent and travels to a model + * provider. Only the desktop application reads it, and it does so at the point + * it builds the provider. Everything *else* a transcription needs comes out of + * the project directory and is safe for anything to read — so it lives here, + * in one call, rather than being re-derived by each caller. + * + * The language reaches the provider as the hint + * (`adr:0008-content-language-is-a-setting-english-by-default`), rather than + * being left to detection. A provider guessing from the first thirty seconds + * of a Portuguese meeting that opened with English pleasantries gets it wrong + * for the whole chunk, and nothing downstream can tell. + */ +export interface TranscriptionInputs { + language: Language; + vocabulary: string[]; +} + +export function transcriptionInputs( + projectRoot: string, + vocabularyLimit = DEFAULT_VOCABULARY_LIMIT, +): TranscriptionInputs { + return { + language: readSettings(projectRoot).language, + vocabulary: projectVocabulary(projectRoot, vocabularyLimit), + }; +} diff --git a/packages/access/src/sources/vocabulary.ts b/packages/access/src/sources/vocabulary.ts new file mode 100644 index 0000000..5029ab7 --- /dev/null +++ b/packages/access/src/sources/vocabulary.ts @@ -0,0 +1,95 @@ +import { readWiki } from "../check/checks.js"; + +/** + * The transcription vocabulary, seeded from the project's own pages (plan 4.10). + * + * **It is what stops the project's own name from coming out wrong.** A speech + * model has never heard "Fenix" and will write "Phoenix", every time, on every + * chunk — and the wiki that gets built from that transcript cites a project + * that does not exist. The names are already in the wiki: they are its page + * titles and the aliases each page lists. + * + * This reads pages and nothing else, so it lives here rather than in + * `@open-wiki/audio` — which would otherwise have to depend on this package to + * get at them, and this package already depends on that one for the time map. + */ + +/** + * Whisper's prompt window holds only its last 224 tokens, so an unbounded list + * from a large wiki cannot all fit. This bounds how many names are *offered*; + * `vocabularyPrompt` in `@open-wiki/audio` bounds what is actually sent, in + * characters, and emits them so the best are the ones nearest the end — which + * is the end the window keeps. + */ +export const DEFAULT_VOCABULARY_LIMIT = 120; + +/** + * No name is this long, and one that is came from a page in a project that + * arrived by clone. `vocabularyPrompt` clips to the same figure on its side; + * the number is repeated rather than imported because reaching into + * `@open-wiki/audio` for it would put a package that spawns subprocesses on + * this module's import graph. + */ +export const MAX_NAME_CHARS = 64; + +/** Words a model already knows, and which crowd out the ones it does not. */ +const COMMON = new Set([ + "index", + "changelog", + "log", + "readme", + "notes", + "meeting", + "project", + "overview", +]); + +/** + * The names in a project's wiki, best first. + * + * "Best" is rarity: a title that is one unusual word is exactly what a model + * gets wrong, and a title that is a sentence is not a name at all. So single + * tokens that are not ordinary words come first, then the rest, and anything + * that reads as prose is dropped. + */ +export function projectVocabulary(projectRoot: string, limit = DEFAULT_VOCABULARY_LIMIT): string[] { + const names: string[] = []; + for (const page of readWiki(projectRoot)) { + const front = page.frontmatter; + if (!front) continue; + if (typeof front["title"] === "string") names.push(front["title"]); + const aliases = front["aliases"]; + if (Array.isArray(aliases)) { + for (const alias of aliases) if (typeof alias === "string") names.push(alias); + } + } + return rankNames(names, limit); +} + +/** Deduplicate, drop what is not a name, and put the rare words first. */ +export function rankNames(names: readonly string[], limit = DEFAULT_VOCABULARY_LIMIT): string[] { + const seen = new Set(); + const kept: string[] = []; + for (const raw of names) { + const name = raw.trim().replace(/\s+/g, " "); + if (!name) continue; + // Four words or more is a page title describing something, not a name a + // model needs help spelling — and it costs as much prompt as four names. + if (name.split(" ").length > 3) continue; + // A name longer than this is not a name. It came from a page in a project + // that arrived by clone, and it would otherwise become a megabyte of form + // field on the upload and a megabyte of argv on the local provider. + if (name.length > MAX_NAME_CHARS) continue; + if (COMMON.has(name.toLowerCase())) continue; + const key = name.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + kept.push(name); + } + kept.sort((a, b) => wordCount(a) - wordCount(b) || a.localeCompare(b)); + return kept.slice(0, limit); +} + +function wordCount(name: string): number { + return name.split(" ").length; +} diff --git a/packages/access/src/store/staleness.ts b/packages/access/src/store/staleness.ts index 3f76f5f..71b7138 100644 --- a/packages/access/src/store/staleness.ts +++ b/packages/access/src/store/staleness.ts @@ -98,4 +98,4 @@ export function pagesEqual(a: string, b: string): boolean { return false; } return stableStringify(x.frontmatter) === stableStringify(y.frontmatter); -} \ No newline at end of file +} diff --git a/packages/access/tests/gate-guard.spec.ts b/packages/access/tests/gate-guard.spec.ts index 0906007..f0e26aa 100644 --- a/packages/access/tests/gate-guard.spec.ts +++ b/packages/access/tests/gate-guard.spec.ts @@ -58,4 +58,4 @@ describe("isConfigWrite (9.6)", () => { expect(isConfigWrite("CLAUDE.md", root)).toBe(true); expect(isConfigWrite("wiki/fenix.md", root)).toBe(false); }); -}); \ No newline at end of file +}); diff --git a/packages/access/tests/sources-manifest.spec.ts b/packages/access/tests/sources-manifest.spec.ts index 67e0c74..d62afb7 100644 --- a/packages/access/tests/sources-manifest.spec.ts +++ b/packages/access/tests/sources-manifest.spec.ts @@ -2,7 +2,13 @@ import { describe, expect, it, beforeEach, afterEach } from "vitest"; import { mkdtempSync, mkdirSync, rmSync, readFileSync, writeFileSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { readManifest, listSources, sourceExists, TakenIdError, type SourceManifest } from "../src/sources/manifest.js"; +import { + readManifest, + listSources, + sourceExists, + TakenIdError, + type SourceManifest, +} from "../src/sources/manifest.js"; import { registerSource } from "../src/sources/register.js"; import { EmptyNameError } from "../src/sources/id.js"; import { OutsideProjectError } from "../src/paths.js"; @@ -116,7 +122,11 @@ describe("readManifest / listSources / sourceExists (3.1 read side)", () => { // as the id `../../elsewhere`. Answering "yes, that exists" would make the // gate an existence oracle for any manifest.json on the machine, and would // let a page cite something that is not a source at all. - writeFileSync(join(root, "manifest.json"), JSON.stringify({ id: "x", title: "outside raw" }), "utf8"); + writeFileSync( + join(root, "manifest.json"), + JSON.stringify({ id: "x", title: "outside raw" }), + "utf8", + ); mkdirSync(join(root, ".claude"), { recursive: true }); writeFileSync(join(root, ".claude", "manifest.json"), JSON.stringify({ id: "y" }), "utf8"); diff --git a/packages/access/tests/sources-state.spec.ts b/packages/access/tests/sources-state.spec.ts index 17ad2ba..980450f 100644 --- a/packages/access/tests/sources-state.spec.ts +++ b/packages/access/tests/sources-state.spec.ts @@ -93,6 +93,40 @@ describe("source state (6.1)", () => { expect(state.progress).toEqual({ done: 1, total: 2 }); }); + it("is still transcribing while chunks remain untried, and says what failed", () => { + // The pipeline (4.9) records a chunk's error and carries on, because 6.3 + // offers "redo only what failed" and that needs the rest attempted. A + // single 429 twelve minutes into a healthy run must not make the source + // read as `failed` — with a progress count that keeps climbing — for the + // remaining forty minutes. + source(root, "weekly", { + kind: "recording", + journal: { + chunks: [{ done: true }, { error: "groq: 429 rate limited" }, { done: false }], + }, + }); + const state = sourceState(root, "weekly"); + expect(state.stage).toBe("transcribing"); + expect(state.error).toContain("429"); + expect(state.progress).toEqual({ done: 1, total: 3 }); + }); + + it("is failed once nothing is left to try", () => { + source(root, "weekly", { + kind: "recording", + journal: { chunks: [{ done: true }, { error: "a" }, { error: "b" }] }, + }); + expect(sourceState(root, "weekly").stage).toBe("failed"); + }); + + it("is failed when the journal itself carries the error", () => { + source(root, "weekly", { + kind: "recording", + journal: { error: "the recording had no audio", chunks: [{ done: false }] }, + }); + expect(sourceState(root, "weekly").stage).toBe("failed"); + }); + it("is not failed once the text landed anyway", () => { // A chunk that failed and was retried successfully leaves its error in // the journal; the text is the thing that says it finished. diff --git a/packages/access/tests/sources-vocabulary.spec.ts b/packages/access/tests/sources-vocabulary.spec.ts new file mode 100644 index 0000000..8666004 --- /dev/null +++ b/packages/access/tests/sources-vocabulary.spec.ts @@ -0,0 +1,114 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { projectVocabulary, rankNames } from "../src/sources/vocabulary.js"; +import { transcriptionInputs } from "../src/sources/transcription.js"; + +let root: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "ow-vocab-")); + mkdirSync(join(root, "wiki"), { recursive: true }); +}); + +afterEach(() => rmSync(root, { recursive: true, force: true })); + +function page(slug: string, front: Record): void { + const yaml = Object.entries(front) + .map(([k, v]) => `${k}: ${JSON.stringify(v)}`) + .join("\n"); + writeFileSync(join(root, "wiki", `${slug}.md`), `---\n${yaml}\n---\n\nbody\n`, "utf8"); +} + +describe("projectVocabulary (4.10)", () => { + it("collects the titles already in the wiki", () => { + // The names are the whole point: a speech model has never heard "Fenix" + // and writes "Phoenix", every time, on every chunk. + page("fenix", { title: "Fenix", aliases: [] }); + page("mateus", { title: "Mateus Andrade", aliases: [] }); + expect(projectVocabulary(root)).toContain("Fenix"); + expect(projectVocabulary(root)).toContain("Mateus Andrade"); + }); + + it("collects the aliases each page lists, which is where the variants live", () => { + page("fenix", { title: "Fenix", aliases: ["Projeto Fenix", "FNX"] }); + const vocabulary = projectVocabulary(root); + expect(vocabulary).toContain("Projeto Fenix"); + expect(vocabulary).toContain("FNX"); + }); + + it("is empty for a project with no pages yet", () => { + expect(projectVocabulary(root)).toEqual([]); + }); + + it("skips a page whose frontmatter will not parse", () => { + writeFileSync(join(root, "wiki", "broken.md"), "no frontmatter here\n", "utf8"); + page("fenix", { title: "Fenix", aliases: [] }); + expect(projectVocabulary(root)).toEqual(["Fenix"]); + }); + + it("bounds the list, because the prompt window holds only so much", () => { + for (let i = 0; i < 50; i++) page(`p${i}`, { title: `Name${i}`, aliases: [] }); + expect(projectVocabulary(root, 10)).toHaveLength(10); + }); +}); + +describe("transcriptionInputs (4.15)", () => { + it("takes the language from the project's settings, not from detection", () => { + writeFileSync( + join(root, "ow.json"), + JSON.stringify({ language: "pt-BR", deleteWavAfterTranscription: true }), + "utf8", + ); + expect(transcriptionInputs(root).language).toBe("pt-BR"); + }); + + it("defaults to English, which is what an unconfigured project produces", () => { + expect(transcriptionInputs(root).language).toBe("en"); + }); + + it("carries the project's names alongside it", () => { + page("fenix", { title: "Fenix", aliases: [] }); + expect(transcriptionInputs(root).vocabulary).toEqual(["Fenix"]); + }); + + it("carries no credential — the CLI and the hooks must never read one", () => { + // `config/secrets.ts`: their stderr is consumed by an agent and travels to + // a model provider. Only the desktop application reads the key. + expect(Object.keys(transcriptionInputs(root))).toEqual(["language", "vocabulary"]); + }); +}); + +describe("rankNames", () => { + it("removes duplicates however they were cased", () => { + expect(rankNames(["Fenix", "fenix", "FENIX"])).toEqual(["Fenix"]); + }); + + it("puts the single unusual words first, which are what a model gets wrong", () => { + expect(rankNames(["Mateus Andrade", "Fenix"])).toEqual(["Fenix", "Mateus Andrade"]); + }); + + it("drops a title that is a sentence rather than a name", () => { + // Four words is a page describing something. It costs as much prompt as + // four names and helps with none of them. + expect(rankNames(["How the dispatcher routes requests"])).toEqual([]); + }); + + it("drops words a model already knows", () => { + expect(rankNames(["index", "Changelog", "Fenix"])).toEqual(["Fenix"]); + }); + + it("collapses the whitespace inside a name", () => { + expect(rankNames([" Mateus Andrade "])).toEqual(["Mateus Andrade"]); + }); + + it("drops an empty name", () => { + expect(rankNames(["", " "])).toEqual([]); + }); + + it("is stable, so two runs seed the same prompt", () => { + const names = ["Zeta", "Alpha", "Beta Gamma"]; + expect(rankNames(names)).toEqual(rankNames([...names].reverse())); + }); +}); diff --git a/packages/access/tests/store-staleness.spec.ts b/packages/access/tests/store-staleness.spec.ts index 6c2cb7c..f430d83 100644 --- a/packages/access/tests/store-staleness.spec.ts +++ b/packages/access/tests/store-staleness.spec.ts @@ -28,19 +28,23 @@ describe("isStoreOnlyChange (5.8)", () => { }); it("a store-stamped `updated` on a later day is not an external edit", () => { - const author = page("id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-01\nsources: []\nsuperseded-by: \"\""); - const disk = page("id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-02\nsources: []\nsuperseded-by: \"\""); + const author = page( + 'id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-01\nsources: []\nsuperseded-by: ""', + ); + const disk = page( + 'id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-02\nsources: []\nsuperseded-by: ""', + ); expect(isStoreOnlyChange(author, disk)).toBe(true); }); it("a store-appended `sources` is not an external edit (hook path)", () => { // The agent wrote the citation in the body; the store mirrored it into `sources`. const author = page( - "id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-01\nsources: []\nsuperseded-by: \"\"", + 'id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-01\nsources: []\nsuperseded-by: ""', "See src://doc.pdf#p1.\n", ); const disk = page( - "id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-01\nsources:\n - src://doc.pdf#p1\nsuperseded-by: \"\"", + 'id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-01\nsources:\n - src://doc.pdf#p1\nsuperseded-by: ""', "See src://doc.pdf#p1.\n", ); expect(isStoreOnlyChange(author, disk)).toBe(true); @@ -49,7 +53,7 @@ describe("isStoreOnlyChange (5.8)", () => { it("`updated` and `sources` changing together is still store-only", () => { const author = page(FM.replace("updated: 2026-08-01", 'updated: ""')); const disk = page( - "id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-02\nsources:\n - src://doc.pdf#p1\nsuperseded-by: \"\"", + 'id: t:a\ntype: t\ntitle: A\nstatus: active\naliases: []\nupdated: 2026-08-02\nsources:\n - src://doc.pdf#p1\nsuperseded-by: ""', ); expect(isStoreOnlyChange(author, disk)).toBe(true); }); @@ -109,4 +113,4 @@ describe("isStoreOnlyChange (5.8)", () => { expect(isStoreOnlyChange(broken, good)).toBe(false); expect(isStoreOnlyChange(broken, broken)).toBe(true); }); -}); \ No newline at end of file +}); diff --git a/packages/audio/src/absolute.ts b/packages/audio/src/absolute.ts new file mode 100644 index 0000000..400c81a --- /dev/null +++ b/packages/audio/src/absolute.ts @@ -0,0 +1,97 @@ +import type { JournalChunk, TrackName } from "./journal.js"; +import { toWallMs, type TimeMap } from "./timemap.js"; + +/** + * Rebuilding absolute time from a chunk's offsets (plan 4.11). + * + * A provider is handed one chunk and answers about that chunk: a segment it + * reports at two seconds is two seconds into what it was sent, not into the + * recording. Two additions stand between that and a citation: + * + * 1. the chunk's own start in the compressed file, and + * 2. the time map, which puts the compressed instant back on the wall clock — + * adding back the silence 4.6 removed and the pauses the recorder removed + * before that. + * + * Doing only the first is the failure that looks right. Every timestamp after + * the first chunk would be wrong by exactly the length of what came before it, + * and the transcript would still read perfectly. + */ + +export interface TimedPassage { + track: TrackName; + compressedStartNs: number; + compressedEndNs: number; + /** Milliseconds since the epoch — see the note on units in `timemap.ts`. */ + wallStartMs: number; + text: string; +} + +export function absolutePassages(chunk: JournalChunk, map: TimeMap): TimedPassage[] { + if (!chunk.done) return []; + + const segments = chunk.segments ?? []; + if (segments.length === 0) { + // A provider that returned only text still produced evidence. Anchoring it + // at the chunk's start is coarse and honest; dropping it loses the words. + const text = (chunk.text ?? "").trim(); + if (!text) return []; + const passage = passageAt(chunk, map, 0, chunk.compressedEndNs - chunk.compressedStartNs, text); + return passage ? [passage] : []; + } + + const passages: TimedPassage[] = []; + for (const segment of segments) { + const text = segment.text.trim(); + if (!text) continue; + const passage = passageAt(chunk, map, segment.startNs, segment.endNs, text); + if (passage) passages.push(passage); + } + return passages; +} + +function passageAt( + chunk: JournalChunk, + map: TimeMap, + startNs: number, + endNs: number, + text: string, +): TimedPassage | null { + // One ceiling, applied once. The chunk's end and the recording's end are the + // same number for the last chunk and the chunk's is lower for every other, + // so taking the smaller of the two and clamping everything against it keeps + // the three returned fields describing one instant. Clamping them separately + // is how a passage ends up claiming to start at 28 s, end at 25 s, and carry + // a wall time belonging to neither. + const ceiling = Math.min(chunk.compressedEndNs, map.compressedDurationNs); + const rawStart = chunk.compressedStartNs + startNs; + + // Whisper over-runs: asked about ten seconds it sometimes answers about + // eleven. A segment that merely *ends* past the chunk is clipped — the words + // are real and they started here. A segment that *begins* past it is not + // this chunk's at all: the next chunk covers that second and will return the + // same words, and clipping it to a zero-length passage on the boundary is + // how they end up in the timeline twice. + if (rawStart >= ceiling && ceiling > chunk.compressedStartNs) return null; + + const start = clamp(rawStart, chunk.compressedStartNs, ceiling); + const end = clamp(chunk.compressedStartNs + endNs, start, ceiling); + + const wallStartMs = toWallMs(map, start); + // An instant the map refuses has no place in the timeline. Answering with + // the nearest one it accepts would be the map lying, which is the whole + // thing this module exists not to do. + if (wallStartMs === null) return null; + + return { + track: chunk.track, + compressedStartNs: start, + compressedEndNs: end, + wallStartMs, + text, + }; +} + +function clamp(value: number, low: number, high: number): number { + return Math.min(Math.max(value, low), high); +} diff --git a/packages/audio/src/index.ts b/packages/audio/src/index.ts index c3e01c7..48d7185 100644 --- a/packages/audio/src/index.ts +++ b/packages/audio/src/index.ts @@ -15,3 +15,7 @@ export * from "./preprocess.js"; export * from "./recording.js"; export * from "./silence.js"; export * from "./timemap.js"; +export * from "./absolute.js"; +export * from "./journal.js"; +export * from "./transcribe.js"; +export * from "./stt/index.js"; diff --git a/packages/audio/src/journal.ts b/packages/audio/src/journal.ts new file mode 100644 index 0000000..4c82344 --- /dev/null +++ b/packages/audio/src/journal.ts @@ -0,0 +1,259 @@ +import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { ProviderName, SttSegment } from "./stt/provider.js"; +import type { Chunk } from "./timemap.js"; + +/** + * The transcription journal — `adr:0012-transcription-is-a-journalled-serial-pipeline`. + * + * Transcribing an hour of meeting is the one operation in this product that + * costs money, takes real time, and can be interrupted halfway. The journal is + * what makes an interrupted run cost nothing to finish: every chunk's result is + * on disk before the next one starts, so an application killed mid-run loses at + * most the chunk in flight. + * + * It lives in the recording's own directory, beside the audio it describes, so + * an interrupted recording is one directory a person can inspect or delete. + * + * **`sources/state.ts` reads this file too**, for the progress a sources screen + * renders, and it does so through its own idea of the shape — `chunks[].done` + * and `chunks[].error`. That is a contract between two modules and not an + * accident; a field rename here changes what a stalled recording looks like + * there. + */ + +export type TrackName = "mic" | "system"; + +/** + * Both tracks are transcribed. 4.12 labels a passage `me` or `remote` by the + * track it came from, which is only possible if each was read on its own — + * and `adr:0006-opus-as-the-provenance-format` keeps them separate precisely + * so that a recording can be re-read with better attribution later. + */ +export const TRACKS: readonly TrackName[] = ["mic", "system"]; + +export const JOURNAL_FILE = "journal.json"; + +/** One unit of work: one chunk of one track. */ +export interface JournalChunk { + /** Unique across the journal. What a progress count addresses. */ + index: number; + track: TrackName; + compressedStartNs: number; + compressedEndNs: number; + done: boolean; + text?: string; + segments?: SttSegment[]; + error?: string; +} + +export interface Journal { + version: 1; + provider: ProviderName; + model: string; + language: string; + chunks: JournalChunk[]; +} + +/** What a resume expects the journal to describe. */ +export interface JournalExpectation { + provider: ProviderName; + model: string; + chunks: readonly Chunk[]; + tracks?: readonly TrackName[]; + /** + * The content language now configured. `adr:0012` names the provider, the + * model and the boundaries; this is the same class of mismatch and is + * checked with them — resuming a `pt-BR` journal after the setting moved to + * English produces one transcript in two languages, which reads as one. + */ + language?: string; +} + +export type JournalMatch = { ok: true } | { ok: false; reason: string }; + +/** A fresh journal: every chunk of every track, nothing done. */ +export function planJournal(expected: JournalExpectation, language: string): Journal { + const tracks = expected.tracks ?? TRACKS; + const chunks: JournalChunk[] = []; + for (const track of tracks) { + for (const chunk of expected.chunks) { + chunks.push({ + index: chunks.length, + track, + compressedStartNs: chunk.compressedStartNs, + compressedEndNs: chunk.compressedEndNs, + done: false, + }); + } + } + return { version: 1, provider: expected.provider, model: expected.model, language, chunks }; +} + +/** + * Whether a journal describes the same work (plan 4.17). + * + * Resuming across a changed segmentation stitches text from two different cuts + * into one timeline — which produces a plausible, readable, wrong result with + * correct-looking timestamps, and nothing downstream can tell. The same goes + * for a changed provider or model: two models' output in one transcript reads + * as one voice. + * + * So this refuses and the caller offers a clean restart, rather than guessing. + */ +export function journalMatches(journal: Journal, expected: JournalExpectation): JournalMatch { + if (journal.provider !== expected.provider) { + return { + ok: false, + reason: + `this journal was written by ${journal.provider} and the provider is now ` + + `${expected.provider} — resuming would put two models' output in one timeline`, + }; + } + if (journal.model !== expected.model) { + return { + ok: false, + reason: + `this journal was written by model ${journal.model} and the model is now ` + + `${expected.model} — resuming would put two models' output in one timeline`, + }; + } + if (expected.language !== undefined && journal.language !== expected.language) { + return { + ok: false, + reason: + `this journal was written for ${journal.language} and the content language is now ` + + `${expected.language} — resuming would produce one transcript in two languages`, + }; + } + const tracks = expected.tracks ?? TRACKS; + const wanted = planJournal(expected, journal.language).chunks; + if (journal.chunks.length !== wanted.length) { + // Counted in cuts, not in units of work: the user recognises "the + // recording cuts into 3 chunks", not the 6 requests that makes over two + // tracks. + return { + ok: false, + reason: + `this journal covers ${journal.chunks.length / tracks.length} chunks and the ` + + `recording now cuts into ${expected.chunks.length} — the boundaries moved`, + }; + } + for (const [i, chunk] of journal.chunks.entries()) { + const want = wanted[i]!; + if ( + chunk.track !== want.track || + chunk.compressedStartNs !== want.compressedStartNs || + chunk.compressedEndNs !== want.compressedEndNs + ) { + return { + ok: false, + reason: + `chunk ${i} of this journal covers a different stretch of the recording than the ` + + `current boundaries do — every offset inside it would mean something else`, + }; + } + } + return { ok: true }; +} + +/** What still has to be sent: never attempted, or attempted and failed. */ +export function pendingChunks(journal: Journal): JournalChunk[] { + return journal.chunks.filter((chunk) => !chunk.done); +} + +/** True when every unit succeeded — what 4.14 checks before deleting the WAV. */ +export function isComplete(journal: Journal): boolean { + return journal.chunks.length > 0 && journal.chunks.every((chunk) => chunk.done); +} + +export function journalPath(dir: string): string { + return join(dir, JOURNAL_FILE); +} + +/** + * Read the journal, or `null` when there is not a usable one. + * + * Every failure reads as absent: no file, unparseable, or parsed into + * something that is not a journal. A cast is not a check, and this one decides + * whether a paid-for hour of transcription is resumed or thrown away. + */ +export function readJournal(dir: string): Journal | null { + try { + const file = journalPath(dir); + if (!existsSync(file)) return null; + const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); + return isJournal(parsed) ? parsed : null; + } catch { + return null; + } +} + +/** + * Write the journal through a temporary file and a rename. + * + * It is rewritten after every chunk, which makes it the file in this product + * most likely to be caught mid-write by a machine going down — and a truncated + * journal reads as no journal, which throws away everything already paid for. + */ +export function writeJournal(dir: string, journal: Journal): void { + const target = journalPath(dir); + const temp = `${target}.tmp`; + try { + writeFileSync(temp, `${JSON.stringify(journal, null, 2)}\n`, "utf8"); + renameSync(temp, target); + } catch (e) { + rmSync(temp, { force: true }); + throw e; + } +} + +/** + * Whether a parsed value is a journal this version can act on. + * + * It validates the content fields as well as the scheduling ones, which is + * not tidiness. `text` and `segments` are what end up in the timeline and then + * in a wiki page, carrying wall-clock provenance derived from the real time + * map — and a journal marked complete makes 4.14 delete 690 MB of source audio + * without a provider ever having been called. A guard that returns + * `value is Journal` while never looking at those three fields hands every + * consumer a `string` that is a number and an array that is a string. + */ +export function isJournal(value: unknown): value is Journal { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const journal = value as Partial; + if (journal.version !== 1) return false; + if (journal.provider !== "groq" && journal.provider !== "whispercpp") return false; + if (typeof journal.model !== "string" || typeof journal.language !== "string") return false; + if (!Array.isArray(journal.chunks)) return false; + if (journal.chunks.length > MAX_CHUNKS) return false; + return journal.chunks.every(isJournalChunk); +} + +/** + * A day of audio at the 10-minute chunks 4.7 plans, over two tracks, is under + * 300. The bound is here because the array sizes every loop that walks it. + */ +const MAX_CHUNKS = 10_000; + +function isJournalChunk(chunk: unknown): chunk is JournalChunk { + if (typeof chunk !== "object" || chunk === null) return false; + const c = chunk as Partial; + if (!Number.isFinite(c.index)) return false; + if (c.track !== "mic" && c.track !== "system") return false; + if (!Number.isFinite(c.compressedStartNs) || !Number.isFinite(c.compressedEndNs)) return false; + if (typeof c.done !== "boolean") return false; + if (c.text !== undefined && typeof c.text !== "string") return false; + if (c.error !== undefined && typeof c.error !== "string") return false; + if (c.segments !== undefined) { + if (!Array.isArray(c.segments)) return false; + if (!c.segments.every(isSegment)) return false; + } + return true; +} + +function isSegment(segment: unknown): segment is SttSegment { + if (typeof segment !== "object" || segment === null) return false; + const s = segment as Partial; + return Number.isFinite(s.startNs) && Number.isFinite(s.endNs) && typeof s.text === "string"; +} diff --git a/packages/audio/src/stt/groq.ts b/packages/audio/src/stt/groq.ts new file mode 100644 index 0000000..98e1669 --- /dev/null +++ b/packages/audio/src/stt/groq.ts @@ -0,0 +1,195 @@ +import { + FLAC_16K, + secondsToNs, + SttError, + vocabularyPrompt, + type AudioFormat, + type SttProvider, + type SttRequest, + type SttResult, + type SttSegment, +} from "./provider.js"; + +/** + * Groq `whisper-large-v3-turbo` — the default provider (`docs/stack.md`): + * ~US$0.04 an hour, ~228x real time, multilingual, which is what lets the + * content language be a setting rather than a fixed choice. + * + * It is the only credential the application holds, so this is the one module + * in the package that sends anything anywhere. `fetch` is injected: a test + * that reaches the network is a test that fails when somebody is on a train. + */ + +export const GROQ_URL = "https://api.groq.com/openai/v1/audio/transcriptions"; +export const GROQ_MODEL = "whisper-large-v3-turbo"; + +export type FetchLike = (url: string, init: RequestInit) => Promise; + +export interface GroqOptions { + apiKey: string; + model?: string; + baseUrl?: string; + fetch?: FetchLike; + /** Attempts per chunk, including the first. */ + attempts?: number; + /** Injected so a retry test does not actually wait. */ + sleep?: (ms: number) => Promise; + /** How long one attempt may take. A stalled socket must not hang the run. */ + timeoutMs?: number; +} + +/** + * A ten-minute chunk returns in about three seconds at Groq's ~228x real time. + * Five minutes is far beyond slow and well short of forever — which is what a + * request with no timeout is, and what `transcribeRecording` would wait for + * with no journal write and no progress in the meantime. + */ +export const DEFAULT_REQUEST_TIMEOUT_MS = 5 * 60 * 1000; + +/** Enough of an error body to say what went wrong; not enough to be a problem. */ +const MAX_ERROR_BODY_BYTES = 8 * 1024; + +export class InsecureEndpointError extends Error { + constructor(url: string) { + super(`refusing to send the transcription credential to ${url}: it is not https`); + this.name = "InsecureEndpointError"; + } +} + +/** Whisper's `language` is ISO 639-1, so a region tag has to lose its region. */ +export function toIso639(language: string): string { + return language.split("-")[0]!.toLowerCase(); +} + +interface VerboseJson { + text?: string; + segments?: Array<{ start?: number; end?: number; text?: string }>; +} + +export function createGroqProvider(options: GroqOptions): SttProvider { + const model = options.model ?? GROQ_MODEL; + const baseUrl = options.baseUrl ?? GROQ_URL; + // The credential rides on every request, so the endpoint is checked once, + // here, rather than trusted because it usually comes from a constant. + if (!baseUrl.startsWith("https://")) throw new InsecureEndpointError(baseUrl); + const doFetch = options.fetch ?? ((url, init) => fetch(url, init)); + const attempts = Math.max(1, options.attempts ?? 3); + const sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))); + const timeoutMs = options.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + + const audioFormat: AudioFormat = FLAC_16K; + + async function once(request: SttRequest): Promise { + const form = new FormData(); + form.append("file", new Blob([request.audio as Uint8Array]), request.filename); + form.append("model", model); + form.append("response_format", "verbose_json"); + form.append("language", toIso639(request.language)); + const prompt = vocabularyPrompt(request.vocabulary); + if (prompt) form.append("prompt", prompt); + + const response = await doFetch(baseUrl, { + method: "POST", + // The key goes in a header and never into the body or the URL, so it + // cannot end up in a log line. `redirect: "error"` is what makes it not + // end up in a redirect either: undici happens to strip `Authorization` + // across origins today, but a transcription endpoint has no business + // redirecting, and this makes that a property of the code rather than of + // whichever fetch implementation is underneath. + redirect: "error", + headers: { authorization: `Bearer ${options.apiKey}` }, + body: form, + signal: AbortSignal.timeout(timeoutMs), + }); + + if (!response.ok) { + const detail = await safeText(response); + // 429 and 5xx are worth another attempt; 401 and 400 are not, and + // retrying them just spends three times as long saying the same thing. + const retryable = response.status === 429 || response.status >= 500; + throw new SttError(`groq returned ${response.status}: ${detail}`, retryable); + } + + return parseVerboseJson((await response.json()) as VerboseJson); + } + + return { + name: "groq", + model, + audioFormat, + async transcribe(request) { + let last: unknown; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + return await once(request); + } catch (e) { + last = e; + if (!isRetryable(e) || attempt === attempts) break; + // Exponential, so a rate limit that needs a moment gets one rather + // than three requests in the same second. + await sleep(2 ** (attempt - 1) * 1000); + } + } + throw last; + }, + }; +} + +/** The provider's answer, with its seconds turned into this package's nanoseconds. */ +export function parseVerboseJson(body: VerboseJson): SttResult { + const segments: SttSegment[] = []; + for (const raw of body.segments ?? []) { + const text = (raw.text ?? "").trim(); + if (!text) continue; + segments.push({ + startNs: secondsToNs(raw.start ?? 0), + endNs: secondsToNs(raw.end ?? raw.start ?? 0), + text, + }); + } + return { segments, text: (body.text ?? segments.map((s) => s.text).join(" ")).trim() }; +} + +/** + * Whether another attempt is worth making. + * + * `adr:0012` names "the network drops between chunk four and chunk five" as a + * motivating failure, and a `fetch` that rejects for that reason throws a + * `TypeError` or an `AbortError` — not an `SttError`. Retrying only on the + * errors this module raised itself would give a 503 three attempts and the + * most common transient failure exactly one. + * + * `InsecureEndpointError` never reaches here: it is thrown when the provider is + * built, not when a request is made. + */ +function isRetryable(error: unknown): boolean { + if (error instanceof SttError) return error.retryable; + // Anything else came out of `fetch` — a socket, a name lookup, a timeout. + return true; +} + +async function safeText(response: Response): Promise { + try { + const body = response.body; + if (!body) return "(no body)"; + // Read up to a cap rather than reading it whole and slicing: a response + // that streams indefinitely would otherwise be consumed to exhaustion + // before the first character was thrown away. + const reader = body.getReader(); + const parts: string[] = []; + let size = 0; + try { + while (size < MAX_ERROR_BODY_BYTES) { + const { done, value } = await reader.read(); + if (done || !value) break; + size += value.byteLength; + parts.push(new TextDecoder().decode(value, { stream: true })); + } + } finally { + await reader.cancel().catch(() => {}); + } + return parts.join("").slice(0, 500); + } catch { + return "(no body)"; + } +} diff --git a/packages/audio/src/stt/index.ts b/packages/audio/src/stt/index.ts new file mode 100644 index 0000000..23ddd99 --- /dev/null +++ b/packages/audio/src/stt/index.ts @@ -0,0 +1,109 @@ +import { spawn } from "node:child_process"; +import { createGroqProvider, type GroqOptions } from "./groq.js"; +import { createWhisperCppProvider, type WhisperCppOptions } from "./whispercpp.js"; +import type { ProviderName, SttProvider } from "./provider.js"; + +/** + * Choosing the provider from configuration (plan 4.8). + * + * The two are swapped here and nowhere else, so the pipeline above never + * branches on which one it got. That is what makes "the audio never leaves the + * machine" a setting rather than a fork of the code. + */ + +export interface SttConfig { + provider: ProviderName; + /** Groq's, and the application's only credential. Unused by whisper.cpp. */ + apiKey?: string; + model?: string; + /** whisper.cpp only: where its executable and GGML model are. */ + whisperExe?: string; + whisperModel?: string; +} + +export class MissingCredentialError extends Error { + constructor() { + super( + "the Groq provider needs an API key. Add one in the application's settings, " + + "or choose whisper.cpp, which needs no credential and keeps the audio here.", + ); + this.name = "MissingCredentialError"; + } +} + +export class MissingWhisperPathError extends Error { + constructor() { + super( + "the whisper.cpp provider needs the path to its executable and its model. " + + "Neither is bundled — they are large and the choice of model is the user's.", + ); + this.name = "MissingWhisperPathError"; + } +} + +export interface CreateProviderDeps { + groq?: (options: GroqOptions) => SttProvider; + whispercpp?: (options: WhisperCppOptions) => SttProvider; + run?: WhisperCppOptions["run"]; +} + +/** + * Run the local binary. It is the default rather than an injected dependency + * so that a provider can be built from configuration alone — which is the + * whole job of this function. A caller that forgot to pass a spawn seam should + * get a working provider, not a message telling them to fix settings that are + * already right. + */ +function defaultSpawn(): WhisperCppOptions["run"] { + return async (exe, args) => + new Promise((resolvePromise, rejectPromise) => { + const child = spawn(exe, [...args], { windowsHide: true }); + let stderr = ""; + child.stderr.on("data", (d: Buffer) => { + // Bounded for the same reason `spawnFfmpeg` bounds its own: a run over + // a damaged file emits a line per bad frame. + stderr = (stderr + d.toString("utf8")).slice(-MAX_STDERR_CHARS); + }); + child.on("error", rejectPromise); + child.on("close", (code) => resolvePromise({ code: code ?? -1, stderr })); + }); +} + +const MAX_STDERR_CHARS = 64 * 1024; + +export function createProvider(config: SttConfig, deps: CreateProviderDeps = {}): SttProvider { + if (config.provider === "groq") { + if (!config.apiKey) throw new MissingCredentialError(); + const make = deps.groq ?? createGroqProvider; + return make({ + apiKey: config.apiKey, + ...(config.model ? { model: config.model } : {}), + }); + } + if (!config.whisperExe || !config.whisperModel) throw new MissingWhisperPathError(); + const make = deps.whispercpp ?? createWhisperCppProvider; + return make({ + exe: config.whisperExe, + modelPath: config.whisperModel, + run: deps.run ?? defaultSpawn(), + }); +} + +export * from "./provider.js"; +export { + createGroqProvider, + parseVerboseJson, + toIso639, + DEFAULT_REQUEST_TIMEOUT_MS, + GROQ_MODEL, + GROQ_URL, + InsecureEndpointError, +} from "./groq.js"; +export { + createWhisperCppProvider, + parseWhisperJson, + WhisperCppMissingError, + type WhisperCppOptions, + type SpawnLike, +} from "./whispercpp.js"; +export type { GroqOptions, FetchLike } from "./groq.js"; diff --git a/packages/audio/src/stt/provider.ts b/packages/audio/src/stt/provider.ts new file mode 100644 index 0000000..48e1529 --- /dev/null +++ b/packages/audio/src/stt/provider.ts @@ -0,0 +1,154 @@ +/** + * The transcription boundary (plan 4.8). + * + * Two providers sit behind it and they are not variations on one theme: Groq + * is an HTTP call that sends the audio to somebody else's machine, whisper.cpp + * is a subprocess that keeps it here. `docs/stack.md` calls the local one "what + * holds up the privacy argument without rewriting the pipeline" — this + * interface is where that promise is kept, and it is why the pipeline above it + * knows about neither. + * + * The interface is deliberately narrow: one chunk in, its text and its + * segments out. Everything the plan calls for around it — journalling, + * ordering, resuming, reconstructing absolute time — belongs to the pipeline, + * because an adapter that also decided those would have to be written twice + * and would drift on the second. + */ + +/** What a provider wants a chunk delivered as. */ +export interface AudioFormat { + /** The container's extension, without the dot. */ + extension: string; + /** The ffmpeg output options that produce it, after the cut. */ + ffmpegArgs: readonly string[]; +} + +/** + * 16 kHz mono PCM. What whisper.cpp reads natively and the only thing some + * builds read at all — no decode, no dependency, and lossless from the Opus so + * the audio is never encoded twice. + */ +export const WAV_16K: AudioFormat = { + extension: "wav", + ffmpegArgs: ["-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le"], +}; + +/** + * FLAC, for a provider that uploads. + * + * The same samples as `WAV_16K` at roughly half the bytes. That matters + * because a 15-minute chunk of 16 kHz mono PCM is about 28 MB and the upload + * cap is 25 MB (`adr:0006-opus-as-the-provenance-format`) — the one chunk + * length this pipeline is allowed to produce is the one that would not fit. + * Re-encoding the Opus instead would be lossy twice over for no gain. + */ +export const FLAC_16K: AudioFormat = { + extension: "flac", + ffmpegArgs: ["-ac", "1", "-ar", "16000", "-c:a", "flac"], +}; + +export type ProviderName = "groq" | "whispercpp"; + +export interface SttRequest { + /** One chunk, already cut out of the track in the provider's format. */ + audio: Uint8Array; + /** The name the provider sees; its extension is how some pick a demuxer. */ + filename: string; + /** + * The content language, as configured + * (`adr:0008-content-language-is-a-setting-english-by-default`). Sent rather + * than left to detection — a provider guessing from thirty seconds of a + * Portuguese meeting that opened in English gets it wrong for the whole + * chunk, and nothing downstream can tell. + */ + language: string; + /** + * Names already in the project's pages (plan 4.10). It is what stops the + * project's own name from coming out wrong. + */ + vocabulary: readonly string[]; +} + +/** One passage, timed **within the chunk**. Absolute time is 4.11's job. */ +export interface SttSegment { + startNs: number; + endNs: number; + text: string; +} + +export interface SttResult { + segments: SttSegment[]; + /** The whole chunk, for a provider or a chunk that produced no segments. */ + text: string; +} + +export interface SttProvider { + readonly name: ProviderName; + /** Recorded in the journal; a change to it refuses a resume (plan 4.17). */ + readonly model: string; + readonly audioFormat: AudioFormat; + transcribe(request: SttRequest): Promise; +} + +export class SttError extends Error { + constructor( + message: string, + readonly retryable: boolean, + ) { + super(message); + this.name = "SttError"; + } +} + +const NS_PER_SECOND = 1_000_000_000; + +/** Seconds as the nanoseconds every offset in this package is in. */ +export function secondsToNs(seconds: number): number { + return Math.round(seconds * NS_PER_SECOND); +} + +/** No single name is this long; one that is came from a hostile or broken page. */ +export const MAX_NAME_CHARS = 64; + +/** + * About 250 tokens of names — a little over Whisper's window, so the budget is + * spent rather than wasted, and bounded in *characters* because that is what + * actually breaks: a multi-megabyte page title becomes a multi-megabyte form + * field on the upload, and an argv value past Windows' 32 KB limit on the + * local provider. + */ +export const MAX_PROMPT_CHARS = 1000; + +/** + * The prompt that carries the project's vocabulary. + * + * Both providers take a free-text prompt and use it to bias decoding, so this + * is one string built one way rather than two adapters each inventing a + * format. + * + * **The best names go last.** Whisper's prompt window holds only its *last* + * 224 tokens, so anything that overflows is dropped from the front — and a + * list carefully ordered best-first puts "Fenix" exactly where it gets thrown + * away. `rankNames` orders them best-first because that is the useful order + * for a caller to read; this reverses it, because that is the order the model + * keeps. Getting this backwards degrades the feature precisely on the projects + * with enough pages for it to matter. + */ +export function vocabularyPrompt( + vocabulary: readonly string[], + maxChars = MAX_PROMPT_CHARS, +): string { + const kept: string[] = []; + let used = 0; + for (const raw of vocabulary) { + const name = raw.trim().slice(0, MAX_NAME_CHARS); + if (!name) continue; + const cost = name.length + (kept.length > 0 ? 2 : 0); + if (used + cost > maxChars) break; + used += cost; + kept.push(name); + } + // Taken best-first so the budget buys the best names, emitted worst-first so + // the best are the ones nearest the end. + return kept.reverse().join(", "); +} diff --git a/packages/audio/src/stt/whispercpp.ts b/packages/audio/src/stt/whispercpp.ts new file mode 100644 index 0000000..63999ea --- /dev/null +++ b/packages/audio/src/stt/whispercpp.ts @@ -0,0 +1,137 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { + SttError, + vocabularyPrompt, + WAV_16K, + type SttProvider, + type SttRequest, + type SttResult, + type SttSegment, +} from "./provider.js"; + +/** + * whisper.cpp — the optional local provider, "for anyone who requires that the + * audio never leave the machine" (`docs/stack.md`). + * + * It is a subprocess rather than a request, and it needs no credential at all, + * which is the whole point: choosing it is how a user opts out of the one + * place this product talks to a third party. + * + * The binary and the model are not bundled. They are large, they are a + * per-user choice of size against accuracy, and the installer already carries + * ffmpeg and `recorder.exe`. So this refuses clearly when they are absent + * rather than degrading to the provider the user chose *not* to use. + */ + +export type SpawnLike = ( + exe: string, + args: readonly string[], +) => Promise<{ code: number; stderr: string }>; + +export interface WhisperCppOptions { + /** The `whisper-cli` executable. */ + exe: string; + /** The GGML model file. Recorded in the journal, so a swap refuses a resume. */ + modelPath: string; + run: SpawnLike; +} + +export class WhisperCppMissingError extends Error { + constructor(what: string, path: string) { + super( + `whisper.cpp's ${what} is not at ${path}. The local provider is not bundled — ` + + `install it and point the setting at it, or use the Groq provider.`, + ); + this.name = "WhisperCppMissingError"; + } +} + +/** whisper.cpp's `-oj` output: offsets are milliseconds from the file's start. */ +interface WhisperJson { + transcription?: Array<{ + offsets?: { from?: number; to?: number }; + text?: string; + }>; +} + +export function createWhisperCppProvider(options: WhisperCppOptions): SttProvider { + if (!existsSync(options.exe)) throw new WhisperCppMissingError("executable", options.exe); + if (!existsSync(options.modelPath)) throw new WhisperCppMissingError("model", options.modelPath); + + return { + name: "whispercpp", + // The model file's name, not its path: the path is somebody's home + // directory, and this string goes into a journal committed nowhere but + // read by 4.17 to decide whether a resume is the same work. + model: basenameOf(options.modelPath), + audioFormat: WAV_16K, + + async transcribe(request: SttRequest): Promise { + const dir = mkdtempSync(join(tmpdir(), "ow-whisper-")); + // `basename`, because `filename` is documented as "the name the provider + // sees" and this is an exported provider: nothing here should depend on + // a numeric guard in a different file to know it is not a path. + const input = join(dir, basename(request.filename)); + const outputStem = join(dir, "out"); + try { + writeFileSync(input, request.audio); + const args = [ + "-m", + options.modelPath, + "-f", + input, + "-l", + request.language.split("-")[0]!.toLowerCase(), + "-oj", + "-of", + outputStem, + ]; + const prompt = vocabularyPrompt(request.vocabulary); + if (prompt) args.push("--prompt", prompt); + + const result = await options.run(options.exe, args); + if (result.code !== 0) { + // A local run that failed will fail again on the same input, so the + // journal should record it rather than the pipeline spinning on it. + throw new SttError(`whisper.cpp exited ${result.code}: ${result.stderr}`, false); + } + const file = `${outputStem}.json`; + if (!existsSync(file)) { + throw new SttError("whisper.cpp wrote no transcription", false); + } + return parseWhisperJson(JSON.parse(readFileSync(file, "utf8")) as WhisperJson); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + }; +} + +function basenameOf(path: string): string { + return path.split(/[\\/]/).pop() ?? path; +} + +const NS_PER_MS = 1_000_000; + +export function parseWhisperJson(body: WhisperJson): SttResult { + const segments: SttSegment[] = []; + for (const raw of body.transcription ?? []) { + const text = (raw.text ?? "").trim(); + if (!text) continue; + const from = raw.offsets?.from ?? 0; + segments.push({ + startNs: from * NS_PER_MS, + endNs: (raw.offsets?.to ?? from) * NS_PER_MS, + text, + }); + } + return { + segments, + text: segments + .map((s) => s.text) + .join(" ") + .trim(), + }; +} diff --git a/packages/audio/src/transcribe.ts b/packages/audio/src/transcribe.ts new file mode 100644 index 0000000..0413e72 --- /dev/null +++ b/packages/audio/src/transcribe.ts @@ -0,0 +1,188 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { FfmpegRunner } from "./ffmpeg.js"; +import { runOrThrow } from "./ffmpeg.js"; +import { + journalMatches, + pendingChunks, + planJournal, + readJournal, + writeJournal, + TRACKS, + type Journal, + type JournalChunk, + type TrackName, +} from "./journal.js"; +import { MIC_OPUS, SYSTEM_OPUS } from "./preprocess.js"; +import type { AudioFormat, SttProvider } from "./stt/provider.js"; +import type { TimeMap } from "./timemap.js"; + +/** + * The serial, journalled transcription pipeline + * (`adr:0012-transcription-is-a-journalled-serial-pipeline`, plan 4.9 and 4.17). + * + * One chunk at a time, and the journal written before the next one starts, so + * an application killed mid-run loses at most the chunk in flight. Serial is + * not a simplification: Groq runs at ~228x real time, so parallelism buys + * seconds and multiplies the rate-limit errors the journal exists to survive, + * and whisper.cpp already saturates every core on one chunk. + */ + +const NS_PER_SECOND = 1_000_000_000; + +/** + * How many failures in a row before the run stops. + * + * One bad chunk must not strand the other nineteen — 6.3 offers "redo only + * what failed" and that needs the rest attempted. But a bad credential fails + * every chunk identically, and sending twenty requests to find that out is + * both slow and, on a paid provider, expensive. Three in a row is the shape of + * a systemic failure; one is the shape of a bad chunk. + */ +export const MAX_CONSECUTIVE_FAILURES = 3; + +export class JournalMismatchError extends Error { + constructor(reason: string) { + super( + `${reason}. Start this recording's transcription again from the beginning, ` + + `or put the previous settings back.`, + ); + this.name = "JournalMismatchError"; + } +} + +export interface TranscribeOptions { + /** The recording's directory under `raw/`. */ + dir: string; + provider: SttProvider; + /** `adr:0008` — sent as the hint rather than left to detection. */ + language: string; + /** Names already in the project's pages (plan 4.10). */ + vocabulary?: readonly string[]; + map: TimeMap; + run: FfmpegRunner; + tracks?: readonly TrackName[]; + /** Throw away a journal that no longer matches instead of refusing (4.17). */ + restart?: boolean; + onProgress?: (done: number, total: number) => void; +} + +/** + * Transcribe what is left to transcribe, and return the journal. + * + * Resuming is the default and restarting is the exception: a recording opened + * with a journal continues from where it stopped. A journal that no longer + * describes the same work is refused rather than resumed — see + * `journalMatches`. + */ +export async function transcribeRecording(options: TranscribeOptions): Promise { + const tracks = options.tracks ?? TRACKS; + const expectation = { + provider: options.provider.name, + model: options.provider.model, + chunks: options.map.chunks, + tracks, + language: options.language, + }; + + let journal = readJournal(options.dir); + if (journal) { + const match = journalMatches(journal, expectation); + if (!match.ok) { + if (!options.restart) throw new JournalMismatchError(match.reason); + journal = null; + } + } + if (!journal) { + journal = planJournal(expectation, options.language); + writeJournal(options.dir, journal); + } + + const total = journal.chunks.length; + let consecutiveFailures = 0; + + for (const chunk of pendingChunks(journal)) { + try { + const result = await transcribeChunk(options, chunk); + chunk.text = result.text; + chunk.segments = result.segments; + chunk.done = true; + delete chunk.error; + consecutiveFailures = 0; + } catch (e) { + chunk.done = false; + chunk.error = e instanceof Error ? e.message : String(e); + consecutiveFailures += 1; + } + // Written before the next chunk starts. This line is the whole record. + writeJournal(options.dir, journal); + options.onProgress?.(journal.chunks.filter((c) => c.done).length, total); + if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) break; + } + + return journal; +} + +/** Cut one chunk out of its track and send it. */ +async function transcribeChunk(options: TranscribeOptions, chunk: JournalChunk) { + const format = options.provider.audioFormat; + const dir = mkdtempSync(join(tmpdir(), "ow-chunk-")); + const file = join(dir, `chunk-${chunk.index}.${format.extension}`); + try { + await runOrThrow( + options.run, + extractChunkArgs(join(options.dir, trackFile(chunk.track)), chunk, file, format), + ); + return await options.provider.transcribe({ + audio: readFileSync(file), + filename: `chunk-${chunk.index}.${format.extension}`, + language: options.language, + vocabulary: options.vocabulary ?? [], + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +export function trackFile(track: TrackName): string { + return track === "mic" ? MIC_OPUS : SYSTEM_OPUS; +} + +/** + * The ffmpeg invocation that cuts one chunk out of a track. + * + * `-ss` comes *after* `-i`, which makes the seek accurate rather than + * packet-aligned. A packet-aligned seek is up to 20 ms off, and a chunk + * boundary here sits between two stretches of speech with the silence already + * removed — 20 ms is a clipped syllable at each end, on every chunk. + * + * The chunk is decoded to the provider's format rather than copied. Copying + * would hand the provider Opus, which is what it already is — but cutting Opus + * without re-encoding cannot be accurate, and re-encoding Opus to Opus is lossy + * twice for nothing. + */ +export function extractChunkArgs( + source: string, + chunk: { compressedStartNs: number; compressedEndNs: number }, + output: string, + format: AudioFormat, +): string[] { + return [ + "-hide_banner", + "-nostats", + "-y", + "-i", + source, + "-ss", + seconds(chunk.compressedStartNs), + "-to", + seconds(chunk.compressedEndNs), + ...format.ffmpegArgs, + output, + ]; +} + +function seconds(ns: number): string { + return (ns / NS_PER_SECOND).toFixed(6); +} diff --git a/packages/audio/tests/absolute.spec.ts b/packages/audio/tests/absolute.spec.ts new file mode 100644 index 0000000..d958ff4 --- /dev/null +++ b/packages/audio/tests/absolute.spec.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from "vitest"; +import { absolutePassages } from "../src/absolute.js"; +import type { JournalChunk } from "../src/journal.js"; +import type { TimeMap } from "../src/timemap.js"; + +const SECOND = 1_000_000_000; +const s = (n: number): number => n * SECOND; + +/** + * 20 s captured with 5 s..10 s of shared silence removed, then a minute's + * pause, then 10 s more. Compressed: 0..5 s, 5..15 s, 15..25 s. + */ +const map: TimeMap = { + version: 1, + compressedDurationNs: s(25), + segments: [ + { compressedStartNs: 0, durationNs: s(5), recordedStartNs: 0, wallStartMs: 1_000_000 }, + { compressedStartNs: s(5), durationNs: s(10), recordedStartNs: s(10), wallStartMs: 1_010_000 }, + { compressedStartNs: s(15), durationNs: s(10), recordedStartNs: s(20), wallStartMs: 1_080_000 }, + ], + chunks: [], +}; + +function chunk(over: Partial = {}): JournalChunk { + return { + index: 0, + track: "mic", + compressedStartNs: 0, + compressedEndNs: s(25), + done: true, + segments: [], + ...over, + }; +} + +describe("absolutePassages (4.11)", () => { + it("adds the chunk's offset to a segment timed from the chunk's own start", () => { + // A provider is handed one chunk and answers about that chunk. A segment + // it reports at 2 s in the second chunk is not 2 s into the recording, and + // treating it as one is how every timestamp after the first chunk ends up + // wrong by exactly the length of what came before. + const passages = absolutePassages( + chunk({ + compressedStartNs: s(15), + compressedEndNs: s(25), + segments: [{ startNs: s(2), endNs: s(4), text: "the cutover" }], + }), + map, + ); + expect(passages[0]!.compressedStartNs).toBe(s(17)); + }); + + it("resolves the absolute instant through the time map", () => { + // Compressed 17 s is recorded 22 s, which happened a minute and two + // seconds after the pause resumed. + const passages = absolutePassages( + chunk({ + compressedStartNs: s(15), + compressedEndNs: s(25), + segments: [{ startNs: s(2), endNs: s(4), text: "the cutover" }], + }), + map, + ); + expect(passages[0]!.wallStartMs).toBe(1_082_000); + }); + + it("accounts for the silence that was cut, not only for the chunk offset", () => { + // Compressed 6 s is recorded 11 s. A reconstruction that added the chunk + // offset and stopped would say 6 s after the start, which is inside the + // removed silence — a moment nothing recorded. + const passages = absolutePassages( + chunk({ segments: [{ startNs: s(6), endNs: s(7), text: "after the gap" }] }), + map, + ); + expect(passages[0]!.wallStartMs).toBe(1_011_000); + }); + + it("carries the track, which is what labels me and remote", () => { + const passages = absolutePassages( + chunk({ track: "system", segments: [{ startNs: 0, endNs: SECOND, text: "hello" }] }), + map, + ); + expect(passages[0]!.track).toBe("system"); + }); + + it("keeps the segments in order", () => { + const passages = absolutePassages( + chunk({ + segments: [ + { startNs: 0, endNs: SECOND, text: "first" }, + { startNs: s(2), endNs: s(3), text: "second" }, + ], + }), + map, + ); + expect(passages.map((p) => p.text)).toEqual(["first", "second"]); + }); + + it("clamps a segment that ran past the end of its chunk", () => { + // Whisper over-runs: asked about ten seconds it sometimes answers about + // eleven. Left alone, the eleventh second belongs to the next chunk too, + // and the same words appear twice in the timeline. + const passages = absolutePassages( + chunk({ + compressedStartNs: 0, + compressedEndNs: s(5), + segments: [{ startNs: s(4), endNs: s(9), text: "over the edge" }], + }), + map, + ); + expect(passages[0]!.compressedEndNs).toBe(s(5)); + }); + + it("clamps a segment that would resolve past the end of the recording", () => { + const passages = absolutePassages( + chunk({ + compressedStartNs: s(15), + compressedEndNs: s(25), + segments: [{ startNs: s(9), endNs: s(30), text: "the last word" }], + }), + map, + ); + expect(passages[0]!.wallStartMs).not.toBeNull(); + expect(passages[0]!.compressedEndNs).toBeLessThanOrEqual(s(25)); + }); + + it("drops a segment that begins past the end of its chunk", () => { + // Not the same as one that merely runs over. A segment that *starts* after + // the chunk ends is the next chunk's, and that chunk will return the same + // words — clipping it to a zero-length passage on the boundary is how they + // land in the timeline twice. + const passages = absolutePassages( + chunk({ + compressedStartNs: 0, + compressedEndNs: s(5), + segments: [{ startNs: s(6), endNs: s(7), text: "belongs to the next chunk" }], + }), + map, + ); + expect(passages).toEqual([]); + }); + + it("never emits a passage that ends before it starts", () => { + // Clamping the start against the chunk and the end against the recording + // is how a passage comes to claim it starts at 28 s and ends at 25 s. + const passages = absolutePassages( + chunk({ + compressedStartNs: s(20), + compressedEndNs: s(30), + segments: [{ startNs: s(1), endNs: s(9), text: "past the map" }], + }), + map, + ); + for (const passage of passages) { + expect(passage.compressedEndNs).toBeGreaterThanOrEqual(passage.compressedStartNs); + expect(passage.compressedStartNs).toBeLessThanOrEqual(map.compressedDurationNs); + expect(passage.compressedEndNs).toBeLessThanOrEqual(map.compressedDurationNs); + } + }); + + it("returns nothing rather than a wrong instant when the map refuses one", () => { + const empty = { ...map, segments: [], compressedDurationNs: 0 }; + expect( + absolutePassages(chunk({ segments: [{ startNs: 0, endNs: 1, text: "orphan" }] }), empty), + ).toEqual([]); + }); + + it("drops a segment with nothing in it", () => { + const passages = absolutePassages( + chunk({ segments: [{ startNs: 0, endNs: SECOND, text: " " }] }), + map, + ); + expect(passages).toEqual([]); + }); + + it("returns nothing for a chunk that was never transcribed", () => { + expect(absolutePassages(chunk({ done: false, segments: undefined }), map)).toEqual([]); + }); + + it("falls back to the chunk's whole text when the provider gave no segments", () => { + // A provider that returns only text still produced evidence. Anchoring it + // at the chunk's start is coarse and honest; dropping it loses the words. + const passages = absolutePassages( + chunk({ compressedStartNs: s(15), compressedEndNs: s(25), segments: [], text: "all of it" }), + map, + ); + expect(passages).toEqual([ + { + track: "mic", + compressedStartNs: s(15), + compressedEndNs: s(25), + wallStartMs: 1_080_000, + text: "all of it", + }, + ]); + }); +}); diff --git a/packages/audio/tests/journal.spec.ts b/packages/audio/tests/journal.spec.ts new file mode 100644 index 0000000..5e63788 --- /dev/null +++ b/packages/audio/tests/journal.spec.ts @@ -0,0 +1,323 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + isComplete, + isJournal, + journalMatches, + pendingChunks, + planJournal, + readJournal, + writeJournal, + type JournalExpectation, +} from "../src/journal.js"; +import type { Chunk } from "../src/timemap.js"; + +const SECOND = 1_000_000_000; +const s = (n: number): number => n * SECOND; + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ow-journal-")); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +const chunks: Chunk[] = [ + { index: 0, compressedStartNs: 0, compressedEndNs: s(600) }, + { index: 1, compressedStartNs: s(600), compressedEndNs: s(900) }, +]; + +const expected: JournalExpectation = { + provider: "groq", + model: "whisper-large-v3-turbo", + chunks, +}; + +describe("planJournal", () => { + it("plans one unit of work per chunk per track", () => { + // Both tracks are transcribed: 4.12 labels `me` and `remote` by the track + // a passage came from, which is only possible if each was read on its own. + const journal = planJournal(expected, "pt-BR"); + expect(journal.chunks).toHaveLength(4); + expect(journal.chunks.filter((c) => c.track === "mic")).toHaveLength(2); + expect(journal.chunks.filter((c) => c.track === "system")).toHaveLength(2); + }); + + it("numbers every unit uniquely, which is what a progress count reads", () => { + // `sources/state.ts` counts `chunks.length` and `chunks.filter(c => c.done)` + // to render "4 of 8". Two entries sharing an index would still count, but + // nothing could address one of them. + const journal = planJournal(expected, "en"); + expect(journal.chunks.map((c) => c.index)).toEqual([0, 1, 2, 3]); + }); + + it("records the provider, the model and the language it was planned for", () => { + const journal = planJournal(expected, "es"); + expect(journal.provider).toBe("groq"); + expect(journal.model).toBe("whisper-large-v3-turbo"); + expect(journal.language).toBe("es"); + }); + + it("carries each chunk's boundaries, so a resume can check the segmentation", () => { + const journal = planJournal(expected, "en"); + const mic = journal.chunks.filter((c) => c.track === "mic"); + expect(mic[0]).toMatchObject({ compressedStartNs: 0, compressedEndNs: s(600) }); + expect(mic[1]).toMatchObject({ compressedStartNs: s(600), compressedEndNs: s(900) }); + }); + + it("starts with nothing done", () => { + expect(planJournal(expected, "en").chunks.every((c) => !c.done)).toBe(true); + }); +}); + +describe("journalMatches (4.17)", () => { + it("accepts a journal describing the same work", () => { + expect(journalMatches(planJournal(expected, "en"), expected)).toEqual({ ok: true }); + }); + + it("refuses a journal from a different provider", () => { + // Resuming across providers stitches two models' output into one timeline + // — plausible, readable, and wrong, with correct-looking timestamps. + const journal = planJournal(expected, "en"); + const match = journalMatches(journal, { ...expected, provider: "whispercpp" }); + expect(match.ok).toBe(false); + expect(match.ok === false && match.reason).toMatch(/provider/i); + }); + + it("refuses a journal from a different model", () => { + const journal = planJournal(expected, "en"); + const match = journalMatches(journal, { ...expected, model: "ggml-large-v3.bin" }); + expect(match.ok).toBe(false); + expect(match.ok === false && match.reason).toMatch(/model/i); + }); + + it("refuses a journal whose chunk count no longer matches", () => { + const journal = planJournal(expected, "en"); + const match = journalMatches(journal, { + ...expected, + chunks: [{ index: 0, compressedStartNs: 0, compressedEndNs: s(900) }], + }); + expect(match.ok).toBe(false); + expect(match.ok === false && match.reason).toMatch(/boundar|chunk/i); + }); + + it("refuses a journal whose boundaries moved, even with the same count", () => { + // The dangerous one: same number of chunks, cut in different places. Every + // offset inside every chunk means something else. + const journal = planJournal(expected, "en"); + const match = journalMatches(journal, { + ...expected, + chunks: [ + { index: 0, compressedStartNs: 0, compressedEndNs: s(500) }, + { index: 1, compressedStartNs: s(500), compressedEndNs: s(900) }, + ], + }); + expect(match.ok).toBe(false); + expect(match.ok === false && match.reason).toMatch(/boundar/i); + }); + + it("refuses a journal written for a different content language", () => { + // Same class as the provider and the model: resuming would produce one + // transcript in two languages, which reads as one. + const journal = planJournal(expected, "pt-BR"); + const match = journalMatches(journal, { ...expected, language: "en" }); + expect(match.ok).toBe(false); + expect(match.ok === false && match.reason).toMatch(/language/i); + }); + + it("does not check the language when the caller did not name one", () => { + expect(journalMatches(planJournal(expected, "pt-BR"), expected)).toEqual({ ok: true }); + }); + + it("counts in cuts, not in units of work, when the boundaries moved", () => { + // Two tracks make four units of two chunks. The user recognises the two. + const journal = planJournal(expected, "en"); + const match = journalMatches(journal, { + ...expected, + chunks: [{ index: 0, compressedStartNs: 0, compressedEndNs: s(900) }], + }); + expect(match.ok === false && match.reason).toContain("2 chunks"); + expect(match.ok === false && match.reason).toContain("into 1"); + }); + + it("accepts a journal that is part-done, which is the whole point", () => { + const journal = planJournal(expected, "en"); + journal.chunks[0]!.done = true; + journal.chunks[0]!.text = "hello"; + expect(journalMatches(journal, expected)).toEqual({ ok: true }); + }); +}); + +describe("pendingChunks", () => { + it("is everything never attempted", () => { + expect(pendingChunks(planJournal(expected, "en"))).toHaveLength(4); + }); + + it("excludes what already succeeded, so a resume costs nothing for it", () => { + const journal = planJournal(expected, "en"); + journal.chunks[0]!.done = true; + journal.chunks[2]!.done = true; + expect(pendingChunks(journal).map((c) => c.index)).toEqual([1, 3]); + }); + + it("includes what failed — a failure is work still to do", () => { + const journal = planJournal(expected, "en"); + journal.chunks[0]!.error = "429"; + expect(pendingChunks(journal).map((c) => c.index)).toContain(0); + }); +}); + +describe("isComplete", () => { + it("is true only when every unit succeeded", () => { + const journal = planJournal(expected, "en"); + expect(isComplete(journal)).toBe(false); + for (const chunk of journal.chunks) chunk.done = true; + expect(isComplete(journal)).toBe(true); + }); + + it("is false when one failed, however many succeeded", () => { + // 4.14 checks this before deleting 690 MB of WAV. + const journal = planJournal(expected, "en"); + for (const chunk of journal.chunks) chunk.done = true; + journal.chunks[3]!.done = false; + journal.chunks[3]!.error = "timed out"; + expect(isComplete(journal)).toBe(false); + }); +}); + +describe("readJournal and writeJournal", () => { + it("round-trips", () => { + const journal = planJournal(expected, "pt-BR"); + journal.chunks[0]!.done = true; + journal.chunks[0]!.text = "bom dia"; + writeJournal(dir, journal); + expect(readJournal(dir)).toEqual(journal); + }); + + it("answers null when there is no journal", () => { + expect(readJournal(dir)).toBeNull(); + }); + + it("answers null for a journal that will not parse", () => { + writeFileSync(join(dir, "journal.json"), "{ truncated", "utf8"); + expect(readJournal(dir)).toBeNull(); + }); + + it("answers null for a file that parses into something that is not a journal", () => { + // A cast is not a check, and this one decides whether a paid-for hour of + // transcription is resumed or thrown away. + for (const content of ["{}", "[]", "null", '{"version":2,"chunks":[]}']) { + writeFileSync(join(dir, "journal.json"), content, "utf8"); + expect(readJournal(dir)).toBeNull(); + } + }); + + it("leaves no temporary file behind after a successful write", () => { + // It is rewritten after every chunk, so it is the file in this product + // most likely to be caught mid-write by a machine going down. + writeJournal(dir, planJournal(expected, "en")); + expect(existsSync(join(dir, "journal.json.tmp"))).toBe(false); + }); + + it("leaves the previous journal intact when a write fails", () => { + // The whole reason for the rename: a failed write must not turn a journal + // holding an hour of paid-for transcription into a truncated file, which + // reads as no journal at all. + const first = planJournal(expected, "en"); + first.chunks[0]!.done = true; + first.chunks[0]!.text = "already paid for"; + writeJournal(dir, first); + + const doomed = planJournal(expected, "en") as unknown as { chunks: unknown }; + doomed.chunks = { + toJSON() { + throw new Error("serialisation blew up"); + }, + }; + expect(() => writeJournal(dir, doomed as never)).toThrow(); + + expect(readJournal(dir)).toEqual(first); + expect(existsSync(join(dir, "journal.json.tmp"))).toBe(false); + }); + + it("stays readable by the source-state reader that counts progress", () => { + // `sources/state.ts` reads `chunks[].done` and `chunks[].error` out of this + // file to render the sources screen. It is a separate module with its own + // idea of the shape, and this is the contract between them. + const journal = planJournal(expected, "en"); + journal.chunks[0]!.done = true; + journal.chunks[1]!.error = "boom"; + writeJournal(dir, journal); + const raw = JSON.parse(readFileSync(join(dir, "journal.json"), "utf8")) as { + chunks: Array<{ index: number; done?: boolean; error?: string }>; + }; + expect(raw.chunks.filter((c) => c.done)).toHaveLength(1); + expect(raw.chunks.find((c) => c.error)?.error).toBe("boom"); + }); +}); + +describe("isJournal", () => { + it("accepts what planJournal produced", () => { + expect(isJournal(planJournal(expected, "en"))).toBe(true); + }); + + it("refuses an object that merely parsed", () => { + expect(isJournal({})).toBe(false); + expect(isJournal(null)).toBe(false); + expect(isJournal([])).toBe(false); + }); + + it("refuses a version it was not written to read", () => { + const journal: Record = { ...planJournal(expected, "en"), version: 2 }; + expect(isJournal(journal)).toBe(false); + }); + + it("refuses chunks that are not made of the fields a resume needs", () => { + const journal = planJournal(expected, "en"); + (journal.chunks[0] as { compressedStartNs: unknown }).compressedStartNs = "0"; + expect(isJournal(journal)).toBe(false); + }); + + it("refuses content fields that are not what their type says", () => { + // `text` and `segments` are what end up in the timeline and then in a wiki + // page, carrying wall-clock provenance from the real time map. A guard + // that returns `value is Journal` without looking at them hands every + // consumer a string that is a number. + for (const bad of [ + { text: 5 }, + { error: [] }, + { segments: "abc" }, + { segments: [{ startNs: "0", endNs: 1, text: "x" }] }, + { segments: [{ startNs: 0, endNs: 1 }] }, + ]) { + const journal = planJournal(expected, "en"); + Object.assign(journal.chunks[0]!, bad); + expect(isJournal(journal)).toBe(false); + } + }); + + it("accepts the content fields a real run writes", () => { + const journal = planJournal(expected, "en"); + Object.assign(journal.chunks[0]!, { + done: true, + text: "bom dia", + segments: [{ startNs: 0, endNs: 1, text: "bom dia" }], + }); + journal.chunks[1]!.error = "429"; + expect(isJournal(journal)).toBe(true); + }); + + it("refuses a chunk list too long to be a recording", () => { + const journal = planJournal(expected, "en"); + journal.chunks = Array.from({ length: 10_001 }, (_, i) => ({ + ...journal.chunks[0]!, + index: i, + })); + expect(isJournal(journal)).toBe(false); + }); +}); diff --git a/packages/audio/tests/stt.spec.ts b/packages/audio/tests/stt.spec.ts new file mode 100644 index 0000000..c27b8f7 --- /dev/null +++ b/packages/audio/tests/stt.spec.ts @@ -0,0 +1,443 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, sep } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + createGroqProvider, + createProvider, + createWhisperCppProvider, + FLAC_16K, + GROQ_MODEL, + InsecureEndpointError, + MAX_PROMPT_CHARS, + MissingCredentialError, + MissingWhisperPathError, + parseVerboseJson, + parseWhisperJson, + SttError, + toIso639, + vocabularyPrompt, + WAV_16K, + WhisperCppMissingError, + type FetchLike, + type SpawnLike, + type SttProvider, +} from "../src/stt/index.js"; + +const SECOND = 1_000_000_000; + +/** A `fetch` that answers with the given JSON and records what it was sent. */ +function fakeFetch(body: unknown, status = 200) { + const calls: Array<{ url: string; init: RequestInit }> = []; + const doFetch: FetchLike = async (url, init) => { + calls.push({ url, init }); + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + }; + return { doFetch, calls }; +} + +function formOf(init: RequestInit): FormData { + return init.body as FormData; +} + +describe("vocabularyPrompt (4.10)", () => { + it("joins the names into one prompt both providers can take", () => { + expect(vocabularyPrompt(["Fenix", "Mateus"])).toBe("Mateus, Fenix"); + }); + + it("is empty when the project has no names yet", () => { + expect(vocabularyPrompt([])).toBe(""); + expect(vocabularyPrompt([" "])).toBe(""); + }); + it("puts the best names last, because the window keeps the end", () => { + // Whisper reads only its last 224 tokens. A list ordered best-first puts + // the name that matters exactly where it is dropped. + expect(vocabularyPrompt(["Fenix", "Mateus Andrade"])).toBe("Mateus Andrade, Fenix"); + }); + + it("spends the character budget on the best names", () => { + const prompt = vocabularyPrompt(["Fenix", "Mateus Andrade", "Someone Else"], 20); + expect(prompt).toContain("Fenix"); + expect(prompt.length).toBeLessThanOrEqual(20); + }); + + it("bounds one absurd name rather than sending it whole", () => { + // A page in a project that arrived by clone. Unbounded, it is a megabyte + // of form field on the upload and past Windows' argv limit locally. + const prompt = vocabularyPrompt(["x".repeat(100_000), "Fenix"]); + expect(prompt.length).toBeLessThanOrEqual(MAX_PROMPT_CHARS); + expect(prompt).toContain("Fenix"); + }); + + it("bounds the whole prompt", () => { + const many = Array.from({ length: 5000 }, (_, i) => `name${i}`); + expect(vocabularyPrompt(many).length).toBeLessThanOrEqual(MAX_PROMPT_CHARS); + }); +}); + +describe("toIso639", () => { + it("drops the region, which Whisper's language field does not take", () => { + expect(toIso639("pt-BR")).toBe("pt"); + expect(toIso639("en")).toBe("en"); + expect(toIso639("ES")).toBe("es"); + }); +}); + +describe("the Groq provider (4.8, 4.15)", () => { + const body = { + text: "bom dia", + segments: [{ start: 1.5, end: 2.25, text: " bom dia " }], + }; + + async function transcribeWith(provider: SttProvider, vocabulary: string[] = []) { + return provider.transcribe({ + audio: new Uint8Array([1, 2, 3]), + filename: "chunk-0.flac", + language: "pt-BR", + vocabulary, + }); + } + + it("sends the configured language rather than leaving it to detection", async () => { + // `adr:0008`. A provider guessing from thirty seconds of a Portuguese + // meeting that opened in English gets it wrong for the whole chunk, and + // nothing downstream can tell. + const { doFetch, calls } = fakeFetch(body); + await transcribeWith(createGroqProvider({ apiKey: "k", fetch: doFetch })); + expect(formOf(calls[0]!.init).get("language")).toBe("pt"); + }); + + it("sends the project's names as the prompt", async () => { + const { doFetch, calls } = fakeFetch(body); + await transcribeWith(createGroqProvider({ apiKey: "k", fetch: doFetch }), ["Fenix"]); + expect(formOf(calls[0]!.init).get("prompt")).toBe("Fenix"); + }); + + it("omits the prompt entirely when there are no names", async () => { + const { doFetch, calls } = fakeFetch(body); + await transcribeWith(createGroqProvider({ apiKey: "k", fetch: doFetch })); + expect(formOf(calls[0]!.init).get("prompt")).toBeNull(); + }); + + it("puts the key in a header, never in the body or the URL", async () => { + const { doFetch, calls } = fakeFetch(body); + await transcribeWith(createGroqProvider({ apiKey: "sk-secret", fetch: doFetch })); + expect((calls[0]!.init.headers as Record)["authorization"]).toBe( + "Bearer sk-secret", + ); + expect(calls[0]!.url).not.toContain("sk-secret"); + expect(formOf(calls[0]!.init).get("prompt")).not.toBe("sk-secret"); + }); + + it("asks for the verbose format, which is the one with timings in it", async () => { + const { doFetch, calls } = fakeFetch(body); + await transcribeWith(createGroqProvider({ apiKey: "k", fetch: doFetch })); + expect(formOf(calls[0]!.init).get("response_format")).toBe("verbose_json"); + expect(formOf(calls[0]!.init).get("model")).toBe(GROQ_MODEL); + }); + + it("wants FLAC, because 15 minutes of PCM does not fit under the upload cap", () => { + expect(createGroqProvider({ apiKey: "k" }).audioFormat).toBe(FLAC_16K); + }); + + it("retries a rate limit and then succeeds", async () => { + let attempt = 0; + const doFetch: FetchLike = async () => { + attempt += 1; + if (attempt === 1) return new Response("slow down", { status: 429 }); + return new Response(JSON.stringify(body), { status: 200 }); + }; + const provider = createGroqProvider({ apiKey: "k", fetch: doFetch, sleep: async () => {} }); + await expect(transcribeWith(provider)).resolves.toMatchObject({ text: "bom dia" }); + expect(attempt).toBe(2); + }); + + it("does not retry a rejected credential, which would fail identically three times", async () => { + let attempt = 0; + const doFetch: FetchLike = async () => { + attempt += 1; + return new Response("bad key", { status: 401 }); + }; + const provider = createGroqProvider({ apiKey: "k", fetch: doFetch, sleep: async () => {} }); + await expect(transcribeWith(provider)).rejects.toThrow(SttError); + expect(attempt).toBe(1); + }); + + it("retries a network failure, which is the one adr:0012 names", async () => { + // A `fetch` that rejects — DNS, ECONNRESET, TLS, a timeout — throws a + // TypeError, not an SttError. Retrying only on errors this module raised + // itself gave a 503 three attempts and the most common failure one. + let attempt = 0; + const doFetch: FetchLike = async () => { + attempt += 1; + if (attempt === 1) throw new TypeError("fetch failed"); + return new Response(JSON.stringify(body), { status: 200 }); + }; + const provider = createGroqProvider({ apiKey: "k", fetch: doFetch, sleep: async () => {} }); + await expect(transcribeWith(provider)).resolves.toMatchObject({ text: "bom dia" }); + expect(attempt).toBe(2); + }); + + it("bounds how long one attempt may take", async () => { + let signal: AbortSignal | undefined; + const doFetch: FetchLike = async (_url, init) => { + signal = init.signal ?? undefined; + return new Response(JSON.stringify(body), { status: 200 }); + }; + await transcribeWith(createGroqProvider({ apiKey: "k", fetch: doFetch })); + expect(signal).toBeInstanceOf(AbortSignal); + }); + + it("does not follow a redirect, whatever fetch would have done with the header", async () => { + let init: RequestInit | undefined; + const doFetch: FetchLike = async (_url, i) => { + init = i; + return new Response(JSON.stringify(body), { status: 200 }); + }; + await transcribeWith(createGroqProvider({ apiKey: "k", fetch: doFetch })); + expect(init?.redirect).toBe("error"); + }); + + it("refuses to send the credential over plain http", () => { + expect(() => createGroqProvider({ apiKey: "k", baseUrl: "http://example.test/v1" })).toThrow( + InsecureEndpointError, + ); + }); + + it("gives up after the configured number of attempts", async () => { + let attempt = 0; + const doFetch: FetchLike = async () => { + attempt += 1; + return new Response("busy", { status: 503 }); + }; + const provider = createGroqProvider({ + apiKey: "k", + fetch: doFetch, + attempts: 2, + sleep: async () => {}, + }); + await expect(transcribeWith(provider)).rejects.toThrow(/503/); + expect(attempt).toBe(2); + }); +}); + +describe("parseVerboseJson", () => { + it("turns the provider's seconds into this package's nanoseconds", () => { + const result = parseVerboseJson({ + text: "hi", + segments: [{ start: 1.5, end: 2.25, text: "hi" }], + }); + expect(result.segments).toEqual([{ startNs: 1.5 * SECOND, endNs: 2.25 * SECOND, text: "hi" }]); + }); + + it("trims the padding Whisper puts around every segment", () => { + const result = parseVerboseJson({ segments: [{ start: 0, end: 1, text: " spaced " }] }); + expect(result.segments[0]!.text).toBe("spaced"); + }); + + it("drops a segment with no words in it", () => { + const result = parseVerboseJson({ text: "", segments: [{ start: 0, end: 1, text: " " }] }); + expect(result.segments).toEqual([]); + }); + + it("builds the whole text from the segments when the provider gave none", () => { + const result = parseVerboseJson({ + segments: [ + { start: 0, end: 1, text: "one" }, + { start: 1, end: 2, text: "two" }, + ], + }); + expect(result.text).toBe("one two"); + }); + + it("survives an answer with nothing in it", () => { + expect(parseVerboseJson({})).toEqual({ segments: [], text: "" }); + }); +}); + +describe("parseWhisperJson", () => { + it("reads whisper.cpp's millisecond offsets", () => { + const result = parseWhisperJson({ + transcription: [{ offsets: { from: 1500, to: 2250 }, text: " bom dia " }], + }); + expect(result.segments).toEqual([ + { startNs: 1.5 * SECOND, endNs: 2.25 * SECOND, text: "bom dia" }, + ]); + }); + + it("survives an answer with nothing in it", () => { + expect(parseWhisperJson({})).toEqual({ segments: [], text: "" }); + }); +}); + +describe("createWhisperCppProvider", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ow-whisper-test-")); + writeFileSync(join(dir, "whisper-cli.exe"), ""); + writeFileSync(join(dir, "ggml-large-v3-turbo.bin"), ""); + }); + + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + /** whisper.cpp writing its `-oj` output where it was told to. */ + function localRun(json: unknown, code = 0): SpawnLike { + return async (_exe, args) => { + const stem = args[args.indexOf("-of") + 1]!; + if (code === 0) writeFileSync(`${stem}.json`, JSON.stringify(json), "utf8"); + return { code, stderr: code === 0 ? "" : "whisper: failed" }; + }; + } + + function local(run: SpawnLike) { + return createWhisperCppProvider({ + exe: join(dir, "whisper-cli.exe"), + modelPath: join(dir, "ggml-large-v3-turbo.bin"), + run, + }); + } + + const request = { + audio: new Uint8Array([1, 2, 3]), + filename: "chunk-0.wav", + language: "pt-BR", + vocabulary: ["Fenix"], + }; + + it("refuses clearly when the binary is not installed", () => { + expect(() => + createWhisperCppProvider({ + exe: "C:/nowhere/whisper-cli.exe", + modelPath: "C:/nowhere/model.bin", + run: async () => ({ code: 0, stderr: "" }), + }), + ).toThrow(WhisperCppMissingError); + }); + + it("refuses clearly when the model is not there", () => { + expect(() => + createWhisperCppProvider({ + exe: join(dir, "whisper-cli.exe"), + modelPath: "C:/nowhere/model.bin", + run: async () => ({ code: 0, stderr: "" }), + }), + ).toThrow(/model/); + }); + + it("wants WAV, which is what it reads natively", () => { + expect(WAV_16K.extension).toBe("wav"); + expect(WAV_16K.ffmpegArgs).toContain("pcm_s16le"); + }); + + it("reports the model file's name, not its path", () => { + // The path is somebody's home directory. The name is what 4.17 compares to + // decide whether a resume is the same work. + expect(local(localRun({})).model).toBe("ggml-large-v3-turbo.bin"); + }); + + it("reads back what whisper.cpp wrote", async () => { + const result = await local( + localRun({ transcription: [{ offsets: { from: 1500, to: 2250 }, text: " bom dia " }] }), + ).transcribe(request); + expect(result.segments).toEqual([ + { startNs: 1.5 * SECOND, endNs: 2.25 * SECOND, text: "bom dia" }, + ]); + }); + + it("sends the configured language and the project's names", async () => { + let args: readonly string[] = []; + const run: SpawnLike = async (_exe, a) => { + args = a; + writeFileSync(`${a[a.indexOf("-of") + 1]!}.json`, "{}", "utf8"); + return { code: 0, stderr: "" }; + }; + await local(run).transcribe(request); + expect(args[args.indexOf("-l") + 1]).toBe("pt"); + expect(args[args.indexOf("--prompt") + 1]).toBe("Fenix"); + }); + + it("omits the prompt when the project has no names yet", async () => { + let args: readonly string[] = []; + const run: SpawnLike = async (_exe, a) => { + args = a; + writeFileSync(`${a[a.indexOf("-of") + 1]!}.json`, "{}", "utf8"); + return { code: 0, stderr: "" }; + }; + await local(run).transcribe({ ...request, vocabulary: [] }); + expect(args).not.toContain("--prompt"); + }); + + it("reports a non-zero exit as an error that will not be retried", async () => { + // A local run that failed will fail again on the same input, so the journal + // should record it rather than the pipeline spinning on it. + await expect(local(localRun({}, 2)).transcribe(request)).rejects.toThrow(SttError); + await expect(local(localRun({}, 2)).transcribe(request)).rejects.toThrow(/exited 2/); + }); + + it("reports a run that wrote nothing", async () => { + const run: SpawnLike = async () => ({ code: 0, stderr: "" }); + await expect(local(run).transcribe(request)).rejects.toThrow(/no transcription/); + }); + + it("leaves no audio behind on the machine it was keeping it on", async () => { + let workDir = ""; + const run: SpawnLike = async (_exe, a) => { + const stem = a[a.indexOf("-of") + 1]!; + workDir = stem.slice(0, stem.lastIndexOf(sep)); + writeFileSync(`${stem}.json`, "{}", "utf8"); + return { code: 0, stderr: "" }; + }; + await local(run).transcribe(request); + expect(existsSync(workDir)).toBe(false); + }); +}); + +describe("createProvider (4.8)", () => { + const stub: SttProvider = { + name: "groq", + model: "m", + audioFormat: FLAC_16K, + transcribe: async () => ({ segments: [], text: "" }), + }; + + it("picks Groq from configuration", () => { + const provider = createProvider({ provider: "groq", apiKey: "k" }, { groq: () => stub }); + expect(provider).toBe(stub); + }); + + it("picks whisper.cpp from configuration", () => { + const local = { ...stub, name: "whispercpp" as const }; + const provider = createProvider( + { provider: "whispercpp", whisperExe: "w.exe", whisperModel: "m.bin" }, + { whispercpp: () => local, run: async () => ({ code: 0, stderr: "" }) }, + ); + expect(provider.name).toBe("whispercpp"); + }); + + it("refuses Groq with no credential, and says the local provider needs none", () => { + expect(() => createProvider({ provider: "groq" })).toThrow(MissingCredentialError); + expect(() => createProvider({ provider: "groq" })).toThrow(/whisper\.cpp/); + }); + + it("refuses whisper.cpp with no paths, because neither is bundled", () => { + expect(() => createProvider({ provider: "whispercpp" })).toThrow(MissingWhisperPathError); + }); + + it("passes a configured model through", () => { + let seen = ""; + createProvider( + { provider: "groq", apiKey: "k", model: "whisper-large-v3" }, + { + groq: (o) => { + seen = o.model ?? ""; + return stub; + }, + }, + ); + expect(seen).toBe("whisper-large-v3"); + }); +}); diff --git a/packages/audio/tests/transcribe.spec.ts b/packages/audio/tests/transcribe.spec.ts new file mode 100644 index 0000000..f0c431d --- /dev/null +++ b/packages/audio/tests/transcribe.spec.ts @@ -0,0 +1,418 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { FfmpegResult, FfmpegRunner } from "../src/ffmpeg.js"; +import { readJournal, writeJournal, planJournal, type Journal } from "../src/journal.js"; +import { FLAC_16K, SttError, type SttProvider, type SttResult } from "../src/stt/index.js"; +import { + extractChunkArgs, + JournalMismatchError, + MAX_CONSECUTIVE_FAILURES, + trackFile, + transcribeRecording, +} from "../src/transcribe.js"; +import type { TimeMap } from "../src/timemap.js"; + +const SECOND = 1_000_000_000; +const s = (n: number): number => n * SECOND; + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ow-transcribe-")); + writeFileSync(join(dir, "mic.opus"), ""); + writeFileSync(join(dir, "system.opus"), ""); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +/** One 20 s stretch, cut into two chunks. */ +const map: TimeMap = { + version: 1, + compressedDurationNs: s(20), + segments: [ + { compressedStartNs: 0, durationNs: s(20), recordedStartNs: 0, wallStartMs: 1_000_000 }, + ], + chunks: [ + { index: 0, compressedStartNs: 0, compressedEndNs: s(10) }, + { index: 1, compressedStartNs: s(10), compressedEndNs: s(20) }, + ], +}; + +const okFfmpeg: FfmpegRunner = async (): Promise => { + // The chunk file has to exist for the pipeline to read it back. + return { code: 0, stdout: "", stderr: "" }; +}; + +/** ffmpeg that actually writes the file the pipeline then reads. */ +function writingFfmpeg(): FfmpegRunner { + return async (args) => { + writeFileSync(args[args.length - 1]!, "audio"); + return { code: 0, stdout: "", stderr: "" }; + }; +} + +function provider( + transcribe: (n: number) => Promise, + over: Partial = {}, +): { provider: SttProvider; calls: () => number } { + let n = 0; + return { + provider: { + name: "groq", + model: "whisper-large-v3-turbo", + audioFormat: FLAC_16K, + transcribe: async () => transcribe(n++), + ...over, + }, + calls: () => n, + }; +} + +const said = async (): Promise => ({ + segments: [{ startNs: 0, endNs: SECOND, text: "hello" }], + text: "hello", +}); + +describe("extractChunkArgs", () => { + it("seeks accurately, by putting -ss after the input", () => { + // A packet-aligned seek is up to 20 ms off, and a chunk boundary here sits + // between two stretches of speech with the silence already removed — that + // is a clipped syllable at each end, on every chunk. + const args = extractChunkArgs( + "mic.opus", + { compressedStartNs: s(10), compressedEndNs: s(20) }, + "out.flac", + FLAC_16K, + ); + expect(args.indexOf("-ss")).toBeGreaterThan(args.indexOf("-i")); + }); + + it("cuts exactly the chunk", () => { + const args = extractChunkArgs( + "mic.opus", + { compressedStartNs: s(10), compressedEndNs: s(20) }, + "out.flac", + FLAC_16K, + ); + expect(args[args.indexOf("-ss") + 1]).toBe("10.000000"); + expect(args[args.indexOf("-to") + 1]).toBe("20.000000"); + }); + + it("decodes into the provider's format rather than copying the Opus", () => { + const args = extractChunkArgs( + "mic.opus", + { compressedStartNs: 0, compressedEndNs: 1 }, + "o.flac", + FLAC_16K, + ); + expect(args).not.toContain("copy"); + expect(args).toContain("flac"); + }); +}); + +describe("trackFile", () => { + it("names each track's Opus", () => { + expect(trackFile("mic")).toBe("mic.opus"); + expect(trackFile("system")).toBe("system.opus"); + }); +}); + +describe("transcribeRecording (4.9)", () => { + it("writes the journal after every chunk, before the next one starts", async () => { + // The whole of `adr:0012`: an application killed mid-run loses at most the + // chunk in flight. Asserted by reading the file from inside the provider, + // which is the only moment "before the next one starts" exists. + const seen: number[] = []; + const p = provider(async () => { + seen.push(readJournal(dir)?.chunks.filter((c) => c.done).length ?? -1); + return said(); + }); + await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + // Nothing done before the first, one before the second, and so on. + expect(seen).toEqual([0, 1, 2, 3]); + }); + + it("transcribes both tracks, which is what lets 4.12 label me and remote", async () => { + const p = provider(said); + const journal = await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + expect(journal.chunks.filter((c) => c.track === "mic" && c.done)).toHaveLength(2); + expect(journal.chunks.filter((c) => c.track === "system" && c.done)).toHaveLength(2); + }); + + it("sends one chunk at a time", async () => { + let inFlight = 0; + let most = 0; + const p = provider(async () => { + inFlight += 1; + most = Math.max(most, inFlight); + await Promise.resolve(); + inFlight -= 1; + return said(); + }); + await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + expect(most).toBe(1); + }); + + it("resumes, sending only what has not succeeded", async () => { + const existing = planJournal( + { provider: "groq", model: "whisper-large-v3-turbo", chunks: map.chunks }, + "en", + ); + existing.chunks[0]!.done = true; + existing.chunks[0]!.text = "already paid for"; + existing.chunks[1]!.error = "429"; + writeJournal(dir, existing); + + const p = provider(said); + const journal = await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + // Three sent: the failed one and the two never attempted. Not the one done. + expect(p.calls()).toBe(3); + expect(journal.chunks[0]!.text).toBe("already paid for"); + }); + + it("clears the error on a chunk that succeeded the second time", async () => { + const existing = planJournal( + { provider: "groq", model: "whisper-large-v3-turbo", chunks: map.chunks }, + "en", + ); + existing.chunks[0]!.error = "429"; + writeJournal(dir, existing); + const p = provider(said); + const journal = await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + expect(journal.chunks[0]!.error).toBeUndefined(); + expect(journal.chunks[0]!.done).toBe(true); + }); + + it("records a failure and carries on to the next chunk", async () => { + // 6.3 offers "redo only what failed", and that needs the rest attempted. + const p = provider(async (n) => { + if (n === 0) throw new SttError("that chunk was unreadable", false); + return said(); + }); + const journal = await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + expect(journal.chunks[0]!.error).toMatch(/unreadable/); + expect(journal.chunks.filter((c) => c.done)).toHaveLength(3); + }); + + it("makes a failure durable before the next chunk is sent", async () => { + // The other half of `adr:0012`. A machine that goes down after chunk two + // failed must come back knowing chunk two failed — otherwise the resume + // re-sends everything that came after it as well. + const seen: Array = []; + const p = provider(async (n) => { + seen.push(readJournal(dir)?.chunks[0]?.error); + if (n === 0) throw new SttError("that chunk was unreadable", false); + return said(); + }); + await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + // Nothing before the first attempt; the error is on disk from the second on. + expect(seen[0]).toBeUndefined(); + expect(seen[1]).toMatch(/unreadable/); + }); + + it("stops after a run of failures, rather than paying for twenty of them", async () => { + // A bad credential fails every chunk identically. Three in a row is the + // shape of a systemic failure; one is the shape of a bad chunk. + const p = provider(async () => { + throw new SttError("401 bad key", false); + }); + await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + expect(p.calls()).toBe(MAX_CONSECUTIVE_FAILURES); + }); + + it("reports progress as it goes", async () => { + const seen: Array<[number, number]> = []; + const p = provider(said); + await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + onProgress: (done, total) => seen.push([done, total]), + }); + expect(seen).toEqual([ + [1, 4], + [2, 4], + [3, 4], + [4, 4], + ]); + }); + + it("passes the language and the vocabulary to the provider", async () => { + let language = ""; + let vocabulary: readonly string[] = []; + const p = provider(said); + p.provider.transcribe = async (request) => { + language = request.language; + vocabulary = request.vocabulary; + return said(); + }; + await transcribeRecording({ + dir, + provider: p.provider, + language: "pt-BR", + vocabulary: ["Fenix"], + map, + run: writingFfmpeg(), + }); + expect(language).toBe("pt-BR"); + expect(vocabulary).toEqual(["Fenix"]); + }); + + it("records a failed extraction as the chunk's error rather than throwing", async () => { + const p = provider(said); + const failing: FfmpegRunner = async () => ({ code: 1, stdout: "", stderr: "Invalid data" }); + const journal = await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: failing, + }); + expect(journal.chunks[0]!.error).toMatch(/Invalid data/); + }); + + it("leaves no chunk audio behind", async () => { + const p = provider(said); + await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + // The recording's directory holds what it started with, plus the journal. + const { readdirSync } = await import("node:fs"); + expect(readdirSync(dir).sort()).toEqual(["journal.json", "mic.opus", "system.opus"]); + }); +}); + +describe("transcribeRecording resuming (4.17)", () => { + function journalFor(over: Partial = {}): Journal { + return { + ...planJournal( + { provider: "groq", model: "whisper-large-v3-turbo", chunks: map.chunks }, + "en", + ), + ...over, + }; + } + + it("refuses a journal from another provider rather than stitching two together", async () => { + writeJournal(dir, journalFor({ provider: "whispercpp" })); + const p = provider(said); + await expect( + transcribeRecording({ dir, provider: p.provider, language: "en", map, run: okFfmpeg }), + ).rejects.toThrow(JournalMismatchError); + expect(p.calls()).toBe(0); + }); + + it("refuses a journal from another model", async () => { + writeJournal(dir, journalFor({ model: "ggml-large-v3.bin" })); + const p = provider(said); + await expect( + transcribeRecording({ dir, provider: p.provider, language: "en", map, run: okFfmpeg }), + ).rejects.toThrow(/model/i); + }); + + it("refuses a journal whose boundaries moved", async () => { + const moved: TimeMap = { + ...map, + chunks: [{ index: 0, compressedStartNs: 0, compressedEndNs: s(20) }], + }; + writeJournal(dir, journalFor()); + const p = provider(said); + await expect( + transcribeRecording({ dir, provider: p.provider, language: "en", map: moved, run: okFfmpeg }), + ).rejects.toThrow(JournalMismatchError); + }); + + it("says what to do about it", async () => { + writeJournal(dir, journalFor({ provider: "whispercpp" })); + const p = provider(said); + await expect( + transcribeRecording({ dir, provider: p.provider, language: "en", map, run: okFfmpeg }), + ).rejects.toThrow(/again from the beginning/); + }); + + it("starts over when the caller asked for a restart", async () => { + writeJournal(dir, journalFor({ provider: "whispercpp" })); + const p = provider(said); + const journal = await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + restart: true, + }); + expect(journal.provider).toBe("groq"); + expect(p.calls()).toBe(4); + }); + + it("ignores a journal that will not parse and plans a fresh one", async () => { + writeFileSync(join(dir, "journal.json"), "{ truncated", "utf8"); + const p = provider(said); + const journal = await transcribeRecording({ + dir, + provider: p.provider, + language: "en", + map, + run: writingFfmpeg(), + }); + expect(journal.chunks).toHaveLength(4); + }); +}); diff --git a/plans/open-wiki.md b/plans/open-wiki.md index a458d2c..3c04f5d 100644 --- a/plans/open-wiki.md +++ b/plans/open-wiki.md @@ -143,16 +143,30 @@ fenix/ a project — usually a repository the user alre - `timemap.json` is the artifact, and **5.4's dormant in-range check is live against it** — a citation past the end of a recording is now refused, naming how long the recording runs. A recording with no map yet keeps the weaker check, because an absent map cannot make a citation resolve falsely. - Durations are nanoseconds; **wall-clock instants are milliseconds**. Nanoseconds since the epoch is ~1.75e18 and JavaScript is exact only to 9.007e15, so the unit is chosen where the exactness is free rather than where it merely reads consistently. - Boundaries are snapped onto the 16 kHz output sample grid before the map is built, so the map and the encoder are computed from the same integers rather than agreeing by rounding. -- [ ] 4.8 (Unit) A `SttProvider` interface with `groq` and `whispercpp` adapters, swappable by configuration -- [ ] 4.9 (TDD) Transcribe chunks one at a time, writing each result to the journal before the next starts, so an application killed mid-run loses at most the chunk in flight — `adr:0012-transcription-is-a-journalled-serial-pipeline` -- [ ] 4.10 (Unit) Seed the transcription vocabulary with the names already present in the project's pages — it is what stops the project's own name from coming out wrong -- [ ] 4.11 (TDD) Reconstruct the absolute timestamps from the chunk offset and the time map +- [x] 4.8 (Unit) A `SttProvider` interface with `groq` and `whispercpp` adapters, swappable by configuration + - Each declares the container it wants a chunk in. Groq takes FLAC because 15 minutes of 16 kHz mono PCM is ~28 MB against a 25 MB cap — the one chunk length 4.7 is allowed to produce is the one that would not fit. whisper.cpp takes WAV, which is what it reads natively. + - Neither the whisper.cpp binary nor its model is bundled. They are large and the size-against-accuracy choice is the user's, so the adapter refuses clearly rather than degrading to the provider the user chose *not* to use. +- [x] 4.9 (TDD) Transcribe chunks one at a time, writing each result to the journal before the next starts, so an application killed mid-run loses at most the chunk in flight — `adr:0012-transcription-is-a-journalled-serial-pipeline` + - **Red observed** first: 28 assertion failures across `journal.spec.ts` and `absolute.spec.ts` against signature-only stubs. + - Both tracks are transcribed, so 4.12 can label `me` and `remote` by the track a passage came from. The journal's unit of work is one chunk of one track. + - A failure records and carries on — 6.3 offers "redo only what failed", which needs the rest attempted — but three failures in a row stop the run. That is the shape of a bad credential, and finding it out costs twenty requests on a paid provider otherwise. + - The journal's `chunks[].done` / `chunks[].error` are a **contract with `sources/state.ts`**, which reads the same file to render progress. Honouring it needed a change there: `failed` used to mean "some chunk has an error", which — now that the pipeline records an error and carries on — made a single 429 twelve minutes into a healthy run read as a failed source, with a progress count that kept climbing, for the remaining forty. `failed` now means nothing is left to try, and the error is carried on the in-flight stage too. +- [x] 4.10 (Unit) Seed the transcription vocabulary with the names already present in the project's pages — it is what stops the project's own name from coming out wrong + - Lives in `@open-wiki/access`, because it reads pages. Ranked: single unusual words are what a model gets wrong, and a title that is a sentence is dropped — it costs as much prompt as four names and helps with none of them. + - **The best names go last in the prompt.** Whisper's window holds only its *last* 224 tokens, so a list ordered best-first puts "Fenix" exactly where truncation drops it — the feature would have degraded precisely on the projects with enough pages to matter. A review caught it; the ranking is best-first and the prompt is emitted reversed. The bound is in characters, because that is what actually breaks. +- [x] 4.11 (TDD) Reconstruct the absolute timestamps from the chunk offset and the time map + - Two additions, and doing only the first is the failure that looks right: every timestamp after the first chunk would be wrong by exactly the length of what came before it, and the transcript would still read perfectly. + - Segments are clamped into their chunk. Whisper over-runs — asked about ten seconds it sometimes answers about eleven — and the eleventh second belongs to the next chunk too, which puts the same words in the timeline twice. - [ ] 4.12 (Unit) Merge the two tracks into a `timeline.json` ordered by real time, labelling `me` and `remote` by the track they came from - [ ] 4.13 (Unit) Render the recording's `text.md` from the timeline, with each passage's instant as a provenance anchor - [ ] 4.14 (Unit) Discard the WAV as soon as transcription confirms success, keeping the Opus as the provenance file -- [ ] 4.15 (Unit) Send the configured content language as the transcription hint rather than relying on the provider detecting it — `adr:0008-content-language-is-a-setting-english-by-default` +- [x] 4.15 (Unit) Send the configured content language as the transcription hint rather than relying on the provider detecting it — `adr:0008-content-language-is-a-setting-english-by-default` + - `transcriptionInputs(projectRoot)` reads the language out of `ow.json` and the names out of the wiki, in one call. It deliberately carries **no credential**: `config/secrets.ts` says the CLI, the hooks and the MCP process must not read the key, because their stderr is consumed by an agent and travels to a model provider. Only the desktop application reads it, at the point it builds the provider — which is why there is no `ow transcribe` verb. - [ ] 4.16 (TDD) Ask what is being recorded before capture starts and build the id from it plus the date — `fenix-weekly-2026-07-31`, `-2` for a second the same day — falling back to the timestamp rather than blocking capture on an empty field -- [ ] 4.17 (TDD) Resume from the journal on reopening, sending only what failed or never ran, and refuse a journal whose chunk boundaries, provider or model no longer match rather than stitching two segmentations into one timeline +- [x] 4.17 (TDD) Resume from the journal on reopening, sending only what failed or never ran, and refuse a journal whose chunk boundaries, provider or model no longer match rather than stitching two segmentations into one timeline + - The dangerous mismatch is not a different chunk *count* — it is the same count cut in different places, where every offset inside every chunk means something else and the result reads perfectly. Boundaries are compared one by one, not summed. + - The refusal says what to do about it, and `restart: true` is how a caller takes the offer. + - **The content language is checked with the provider and the model**, which is one more than `adr:0012` lists. It is the same class of mismatch: resuming a `pt-BR` journal after the setting moved to English produces one transcript in two languages, which reads as one. The record left the field recorded and unread; this makes it mean something. - [ ] 4.18 (Unit) Write `timeline.vtt` beside `timeline.json`, so the recording can be followed in any player and taken away if the user stops using this application ## 5 — The wiki as a validated store