feat(audio): the journalled serial transcription pipeline (4.8-4.11, 4.15, 4.17)

packages/access/src/gate/errors.ts

@@ -7,4 +7,4 @@
 export function formatDenial(filePath: string, reasons: string[]): string {
   const bullets = reasons.map((r) => `  - ${r}`).join("\n");
   return `open-wiki refused the write to ${filePath}:\n${bullets}`;
-}
+}

packages/access/src/gate/guard.ts

@@ -33,4 +33,4 @@ export function isConfigWrite(filePath: string, projectRoot: string): boolean {
 /** The reason the hook gives the agent when it refuses a config write (9.6/9.13). */
 export function configWriteReason(filePath: string): string {
   return `open-wiki refuses writes to its own configuration (${filePath}). The gate, .mcp.json and CLAUDE.md are not editable through the agent.`;
-}
+}

packages/access/src/index.ts

@@ -87,6 +87,8 @@ export {
   type SourceState,
   type SourceStage,
 } from "./sources/state.js";
+export { projectVocabulary, rankNames, DEFAULT_VOCABULARY_LIMIT } from "./sources/vocabulary.js";
+export { transcriptionInputs, type TranscriptionInputs } from "./sources/transcription.js";
 export { uploadTextSource, writeSourceText, normaliseText } from "./sources/ingest.js";
 export {
   uploadPdfSource,

packages/access/src/sources/state.ts

@@ -95,16 +95,31 @@ export function sourceState(
     citedBy: [...citedBy],
   };
 
-  if (failed !== undefined && !textReady) {
-    return { ...base, stage: "failed", error: failed, ...progressOf(chunks.length, done) };
+  // A chunk that failed is not a transcription that stopped. The pipeline
+  // (4.9) records a chunk's error and carries on to the next one, because 6.3
+  // offers "redo only what failed" and that needs the rest attempted — so a
+  // single 429 twelve minutes into a healthy run would otherwise make the
+  // source read as `failed`, with a progress count that keeps climbing, for
+  // the rest of the run.
+  //
+  // `failed` means nothing is left to try: every chunk has been attempted and
+  // at least one did not succeed. The error is carried either way, so a caller
+  // showing a source in flight can still say what went wrong on the way.
+  const untried = chunks.some((c) => !c.done && c.error === undefined);
+  const stopped =
+    journal?.error !== undefined || (chunks.length > 0 && !untried && done < chunks.length);
+  const error = failed !== undefined ? { error: failed } : {};
+
+  if (stopped && !textReady) {
+    return { ...base, stage: "failed", ...error, ...progressOf(chunks.length, done) };
   }
   // `textReady` gates "cited" on purpose: a page citing a source whose text
   // never landed is citing something nothing could have read, and reporting
   // that as the last stage of the pipeline would hide it. The citation is still
   // recorded in `citedBy`, so the caller can say both things.
   if (textReady) return { ...base, stage: citedBy.length > 0 ? "cited" : "text-ready" };
   if (chunks.length > 0) {
-    return { ...base, stage: "transcribing", ...progressOf(chunks.length, done) };
+    return { ...base, stage: "transcribing", ...error, ...progressOf(chunks.length, done) };
   }
   return { ...base, stage: "received" };
 }

packages/access/src/sources/transcription.ts

@@ -0,0 +1,35 @@
+import { readSettings, type Language } from "../config/settings.js";
+import { projectVocabulary, DEFAULT_VOCABULARY_LIMIT } from "./vocabulary.js";
+
+/**
+ * What a transcription needs from the project, minus the credential (plan
+ * 4.10 and 4.15).
+ *
+ * **The credential is deliberately not here.** `config/secrets.ts` says it in
+ * as many words: the CLI, the hooks and the MCP process must not read the
+ * secret, because their stderr is consumed by an agent and travels to a model
+ * provider. Only the desktop application reads it, and it does so at the point
+ * it builds the provider. Everything *else* a transcription needs comes out of
+ * the project directory and is safe for anything to read — so it lives here,
+ * in one call, rather than being re-derived by each caller.
+ *
+ * The language reaches the provider as the hint
+ * (`adr:0008-content-language-is-a-setting-english-by-default`), rather than
+ * being left to detection. A provider guessing from the first thirty seconds
+ * of a Portuguese meeting that opened with English pleasantries gets it wrong
+ * for the whole chunk, and nothing downstream can tell.
+ */
+export interface TranscriptionInputs {
+  language: Language;
+  vocabulary: string[];
+}
+
+export function transcriptionInputs(
+  projectRoot: string,
+  vocabularyLimit = DEFAULT_VOCABULARY_LIMIT,
+): TranscriptionInputs {
+  return {
+    language: readSettings(projectRoot).language,
+    vocabulary: projectVocabulary(projectRoot, vocabularyLimit),
+  };
+}

packages/access/src/sources/vocabulary.ts

@@ -0,0 +1,95 @@
+import { readWiki } from "../check/checks.js";
+
+/**
+ * The transcription vocabulary, seeded from the project's own pages (plan 4.10).
+ *
+ * **It is what stops the project's own name from coming out wrong.** A speech
+ * model has never heard "Fenix" and will write "Phoenix", every time, on every
+ * chunk — and the wiki that gets built from that transcript cites a project
+ * that does not exist. The names are already in the wiki: they are its page
+ * titles and the aliases each page lists.
+ *
+ * This reads pages and nothing else, so it lives here rather than in
+ * `@open-wiki/audio` — which would otherwise have to depend on this package to
+ * get at them, and this package already depends on that one for the time map.
+ */
+
+/**
+ * Whisper's prompt window holds only its last 224 tokens, so an unbounded list
+ * from a large wiki cannot all fit. This bounds how many names are *offered*;
+ * `vocabularyPrompt` in `@open-wiki/audio` bounds what is actually sent, in
+ * characters, and emits them so the best are the ones nearest the end — which
+ * is the end the window keeps.
+ */
+export const DEFAULT_VOCABULARY_LIMIT = 120;
+
+/**
+ * No name is this long, and one that is came from a page in a project that
+ * arrived by clone. `vocabularyPrompt` clips to the same figure on its side;
+ * the number is repeated rather than imported because reaching into
+ * `@open-wiki/audio` for it would put a package that spawns subprocesses on
+ * this module's import graph.
+ */
+export const MAX_NAME_CHARS = 64;
+
+/** Words a model already knows, and which crowd out the ones it does not. */
+const COMMON = new Set([
+  "index",
+  "changelog",
+  "log",
+  "readme",
+  "notes",
+  "meeting",
+  "project",
+  "overview",
+]);
+
+/**
+ * The names in a project's wiki, best first.
+ *
+ * "Best" is rarity: a title that is one unusual word is exactly what a model
+ * gets wrong, and a title that is a sentence is not a name at all. So single
+ * tokens that are not ordinary words come first, then the rest, and anything
+ * that reads as prose is dropped.
+ */
+export function projectVocabulary(projectRoot: string, limit = DEFAULT_VOCABULARY_LIMIT): string[] {
+  const names: string[] = [];
+  for (const page of readWiki(projectRoot)) {
+    const front = page.frontmatter;
+    if (!front) continue;
+    if (typeof front["title"] === "string") names.push(front["title"]);
+    const aliases = front["aliases"];
+    if (Array.isArray(aliases)) {
+      for (const alias of aliases) if (typeof alias === "string") names.push(alias);
+    }
+  }
+  return rankNames(names, limit);
+}
+
+/** Deduplicate, drop what is not a name, and put the rare words first. */
+export function rankNames(names: readonly string[], limit = DEFAULT_VOCABULARY_LIMIT): string[] {
+  const seen = new Set<string>();
+  const kept: string[] = [];
+  for (const raw of names) {
+    const name = raw.trim().replace(/\s+/g, " ");
+    if (!name) continue;
+    // Four words or more is a page title describing something, not a name a
+    // model needs help spelling — and it costs as much prompt as four names.
+    if (name.split(" ").length > 3) continue;
+    // A name longer than this is not a name. It came from a page in a project
+    // that arrived by clone, and it would otherwise become a megabyte of form
+    // field on the upload and a megabyte of argv on the local provider.
+    if (name.length > MAX_NAME_CHARS) continue;
+    if (COMMON.has(name.toLowerCase())) continue;
+    const key = name.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    kept.push(name);
+  }
+  kept.sort((a, b) => wordCount(a) - wordCount(b) || a.localeCompare(b));
+  return kept.slice(0, limit);
+}
+
+function wordCount(name: string): number {
+  return name.split(" ").length;
+}

packages/access/src/store/staleness.ts

@@ -98,4 +98,4 @@ export function pagesEqual(a: string, b: string): boolean {
     return false;
   }
   return stableStringify(x.frontmatter) === stableStringify(y.frontmatter);
-}
+}

packages/access/tests/gate-guard.spec.ts

@@ -58,4 +58,4 @@ describe("isConfigWrite (9.6)", () => {
     expect(isConfigWrite("CLAUDE.md", root)).toBe(true);
     expect(isConfigWrite("wiki/fenix.md", root)).toBe(false);
   });
-});
+});

packages/access/tests/sources-manifest.spec.ts

@@ -2,7 +2,13 @@ import { describe, expect, it, beforeEach, afterEach } from "vitest";
 import { mkdtempSync, mkdirSync, rmSync, readFileSync, writeFileSync, existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { readManifest, listSources, sourceExists, TakenIdError, type SourceManifest } from "../src/sources/manifest.js";
+import {
+  readManifest,
+  listSources,
+  sourceExists,
+  TakenIdError,
+  type SourceManifest,
+} from "../src/sources/manifest.js";
 import { registerSource } from "../src/sources/register.js";
 import { EmptyNameError } from "../src/sources/id.js";
 import { OutsideProjectError } from "../src/paths.js";
@@ -116,7 +122,11 @@ describe("readManifest / listSources / sourceExists (3.1 read side)", () => {
     // as the id `../../elsewhere`. Answering "yes, that exists" would make the
     // gate an existence oracle for any manifest.json on the machine, and would
     // let a page cite something that is not a source at all.
-    writeFileSync(join(root, "manifest.json"), JSON.stringify({ id: "x", title: "outside raw" }), "utf8");
+    writeFileSync(
+      join(root, "manifest.json"),
+      JSON.stringify({ id: "x", title: "outside raw" }),
+      "utf8",
+    );
     mkdirSync(join(root, ".claude"), { recursive: true });
     writeFileSync(join(root, ".claude", "manifest.json"), JSON.stringify({ id: "y" }), "utf8");
 

packages/access/tests/sources-state.spec.ts

@@ -93,6 +93,40 @@ describe("source state (6.1)", () => {
       expect(state.progress).toEqual({ done: 1, total: 2 });
     });
 
+    it("is still transcribing while chunks remain untried, and says what failed", () => {
+      // The pipeline (4.9) records a chunk's error and carries on, because 6.3
+      // offers "redo only what failed" and that needs the rest attempted. A
+      // single 429 twelve minutes into a healthy run must not make the source
+      // read as `failed` — with a progress count that keeps climbing — for the
+      // remaining forty minutes.
+      source(root, "weekly", {
+        kind: "recording",
+        journal: {
+          chunks: [{ done: true }, { error: "groq: 429 rate limited" }, { done: false }],
+        },
+      });
+      const state = sourceState(root, "weekly");
+      expect(state.stage).toBe("transcribing");
+      expect(state.error).toContain("429");
+      expect(state.progress).toEqual({ done: 1, total: 3 });
+    });
+
+    it("is failed once nothing is left to try", () => {
+      source(root, "weekly", {
+        kind: "recording",
+        journal: { chunks: [{ done: true }, { error: "a" }, { error: "b" }] },
+      });
+      expect(sourceState(root, "weekly").stage).toBe("failed");
+    });
+
+    it("is failed when the journal itself carries the error", () => {
+      source(root, "weekly", {
+        kind: "recording",
+        journal: { error: "the recording had no audio", chunks: [{ done: false }] },
+      });
+      expect(sourceState(root, "weekly").stage).toBe("failed");
+    });
+
     it("is not failed once the text landed anyway", () => {
       // A chunk that failed and was retried successfully leaves its error in
       // the journal; the text is the thing that says it finished.

packages/access/tests/sources-vocabulary.spec.ts

@@ -0,0 +1,114 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { projectVocabulary, rankNames } from "../src/sources/vocabulary.js";
+import { transcriptionInputs } from "../src/sources/transcription.js";
+
+let root: string;
+
+beforeEach(() => {
+  root = mkdtempSync(join(tmpdir(), "ow-vocab-"));
+  mkdirSync(join(root, "wiki"), { recursive: true });
+});
+
+afterEach(() => rmSync(root, { recursive: true, force: true }));
+
+function page(slug: string, front: Record<string, unknown>): void {
+  const yaml = Object.entries(front)
+    .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
+    .join("\n");
+  writeFileSync(join(root, "wiki", `${slug}.md`), `---\n${yaml}\n---\n\nbody\n`, "utf8");
+}
+
+describe("projectVocabulary (4.10)", () => {
+  it("collects the titles already in the wiki", () => {
+    // The names are the whole point: a speech model has never heard "Fenix"
+    // and writes "Phoenix", every time, on every chunk.
+    page("fenix", { title: "Fenix", aliases: [] });
+    page("mateus", { title: "Mateus Andrade", aliases: [] });
+    expect(projectVocabulary(root)).toContain("Fenix");
+    expect(projectVocabulary(root)).toContain("Mateus Andrade");
+  });
+
+  it("collects the aliases each page lists, which is where the variants live", () => {
+    page("fenix", { title: "Fenix", aliases: ["Projeto Fenix", "FNX"] });
+    const vocabulary = projectVocabulary(root);
+    expect(vocabulary).toContain("Projeto Fenix");
+    expect(vocabulary).toContain("FNX");
+  });
+
+  it("is empty for a project with no pages yet", () => {
+    expect(projectVocabulary(root)).toEqual([]);
+  });
+
+  it("skips a page whose frontmatter will not parse", () => {
+    writeFileSync(join(root, "wiki", "broken.md"), "no frontmatter here\n", "utf8");
+    page("fenix", { title: "Fenix", aliases: [] });
+    expect(projectVocabulary(root)).toEqual(["Fenix"]);
+  });
+
+  it("bounds the list, because the prompt window holds only so much", () => {
+    for (let i = 0; i < 50; i++) page(`p${i}`, { title: `Name${i}`, aliases: [] });
+    expect(projectVocabulary(root, 10)).toHaveLength(10);
+  });
+});
+
+describe("transcriptionInputs (4.15)", () => {
+  it("takes the language from the project's settings, not from detection", () => {
+    writeFileSync(
+      join(root, "ow.json"),
+      JSON.stringify({ language: "pt-BR", deleteWavAfterTranscription: true }),
+      "utf8",
+    );
+    expect(transcriptionInputs(root).language).toBe("pt-BR");
+  });
+
+  it("defaults to English, which is what an unconfigured project produces", () => {
+    expect(transcriptionInputs(root).language).toBe("en");
+  });
+
+  it("carries the project's names alongside it", () => {
+    page("fenix", { title: "Fenix", aliases: [] });
+    expect(transcriptionInputs(root).vocabulary).toEqual(["Fenix"]);
+  });
+
+  it("carries no credential — the CLI and the hooks must never read one", () => {
+    // `config/secrets.ts`: their stderr is consumed by an agent and travels to
+    // a model provider. Only the desktop application reads the key.
+    expect(Object.keys(transcriptionInputs(root))).toEqual(["language", "vocabulary"]);
+  });
+});
+
+describe("rankNames", () => {
+  it("removes duplicates however they were cased", () => {
+    expect(rankNames(["Fenix", "fenix", "FENIX"])).toEqual(["Fenix"]);
+  });
+
+  it("puts the single unusual words first, which are what a model gets wrong", () => {
+    expect(rankNames(["Mateus Andrade", "Fenix"])).toEqual(["Fenix", "Mateus Andrade"]);
+  });
+
+  it("drops a title that is a sentence rather than a name", () => {
+    // Four words is a page describing something. It costs as much prompt as
+    // four names and helps with none of them.
+    expect(rankNames(["How the dispatcher routes requests"])).toEqual([]);
+  });
+
+  it("drops words a model already knows", () => {
+    expect(rankNames(["index", "Changelog", "Fenix"])).toEqual(["Fenix"]);
+  });
+
+  it("collapses the whitespace inside a name", () => {
+    expect(rankNames(["  Mateus   Andrade  "])).toEqual(["Mateus Andrade"]);
+  });
+
+  it("drops an empty name", () => {
+    expect(rankNames(["", "   "])).toEqual([]);
+  });
+
+  it("is stable, so two runs seed the same prompt", () => {
+    const names = ["Zeta", "Alpha", "Beta Gamma"];
+    expect(rankNames(names)).toEqual(rankNames([...names].reverse()));
+  });
+});