feat(access): ingest PDF, DOCX and the raw/_inbox doorway

docs/stack.md

@@ -42,8 +42,9 @@ Each source adapter has a single responsibility — becoming `text.md` with prov
 anchors, and the path stops writing there — see
 `adr:0013-the-project-directory-is-the-unit`.
 
-- **pdf-parse** — text and page boundaries of a PDF; it is the page number that makes the citation possible, and without it the source is of no use.
-- **mammoth** — DOCX to markdown preserving the heading hierarchy, which is the anchor equivalent to a PDF's page.
+- **pdfjs-dist** — text and page boundaries of a PDF; it is the page number that makes the citation possible, and without it the source is of no use. Chosen over `pdf-parse`, which this file named before the adapter was built: `pdf-parse` wraps an old fork of this same engine and hands back the whole document as one string, so the page boundary — the only thing the citation needs — has to be recovered through a render hook. `pdfjs-dist` has `getPage(n).getTextContent()`, which is the boundary directly, and it is the engine upstream maintains. It declares `@napi-rs/canvas` as an *optional* dependency — a native binary, per platform — which rendering needs and text extraction does not; nothing here imports it, and the installer of 10.1 should not carry it.
+- **mammoth** — DOCX to markdown preserving the heading hierarchy, which is what a DOCX carries instead of a PDF's page: the file records no pagination, so the structure is the anchor. Its own markdown writer is deprecated and escapes ordinary prose (`split in two\.`), so the adapter converts mammoth's HTML — a small, predictable subset — itself.
+- **chokidar** — watches `raw/_inbox/` for material an agent dropped there (plan 3.7), and later the project folder (8.10). `fs.watch` alone reports a file the moment it appears, which on a copy is halfway through being written; `awaitWriteFinish` is the part that stops a half-copied PDF becoming a permanently wrong source.
 
 ## MCP server
 

package.json

@@ -4,7 +4,7 @@
   "type": "module",
   "packageManager": "pnpm@10.15.0",
   "engines": {
-    "node": ">=22"
+    "node": ">=22.13.0"
   },
   "scripts": {
     "test": "pnpm -r --if-present run test",

packages/access/package.json

@@ -15,6 +15,9 @@
     "lint": "eslint ."
   },
   "dependencies": {
+    "chokidar": "^5.0.0",
+    "mammoth": "^1.12.0",
+    "pdfjs-dist": "^6.2.108",
     "yaml": "^2.7.0"
   }
 }

packages/access/src/ignore.ts

@@ -11,26 +11,64 @@ import { join } from "node:path";
 export const OPEN_BLOCK = "# >>> open-wiki >>>";
 export const CLOSE_BLOCK = "# <<< open-wiki <<<";
 
-const BLOCK = [
-  OPEN_BLOCK,
+const BODY = [
   "# Recorded audio and .state/ are ignored by default; committing them is",
   "# opt-in. .state/ holds every page as it was before each write, which is",
   "# where a redaction survives the redaction (adr:0013).",
   ".state/",
   "raw/**/*.wav",
   "raw/**/*.opus",
-  CLOSE_BLOCK,
-].join("\n");
+  "# raw/_inbox/ is a doorway, emptied by ingestion (plan 3.7). What sits in it",
+  "# has not been read yet, and committing unreviewed material is not something",
+  "# to do by default; a file that became a source is committed as that source.",
+  "raw/_inbox/",
+  "#",
+  "# Everything between the two markers is managed: it is rewritten whenever the",
+  "# project is scaffolded, so a rule added in a later version reaches a project",
+  "# created by an earlier one. To commit something this ignores, put a negation",
+  "# *below* the closing marker — git takes the last matching pattern, and a line",
+  "# outside the block is never touched.",
+];
+
+const BLOCK = [OPEN_BLOCK, ...BODY, CLOSE_BLOCK].join("\n");
 
 /**
- * Writes the managed block into `<project>/.gitignore`, idempotently. A block
- * already present is left untouched (the user may have opted in by editing it),
- * and any other content is preserved.
+ * Writes the managed block into `<project>/.gitignore`. Content outside the
+ * markers is preserved exactly; content between them is replaced.
+ *
+ * **Replaced, not skipped.** Leaving an existing block untouched meant a rule
+ * added later never reached a project scaffolded earlier — and `scaffold` runs
+ * again on an existing project, so those projects got the new *directory*
+ * without the new ignore rule. For `raw/_inbox/` that is precisely backwards:
+ * the doorway appears, and the unreviewed material an agent drops in it is
+ * git-visible in exactly the projects that already existed.
+ *
+ * Opting in is still supported, and is now a thing the file can express rather
+ * than a thing the tool has to infer from an edit it cannot tell from a
+ * mistake: a negation below the closing marker wins, because git takes the last
+ * matching pattern.
  */
 export function writeIgnore(projectRoot: string): void {
   const file = join(projectRoot, ".gitignore");
   const existing = existsSync(file) ? readFileSync(file, "utf8") : "";
-  if (existing.includes(OPEN_BLOCK)) return; // already managed — do not clobber
-  const body = existing.length === 0 ? BLOCK : `${existing.trimEnd()}\n\n${BLOCK}\n`;
-  writeFileSync(file, body, "utf8");
+
+  if (existing === "") {
+    writeFileSync(file, `${BLOCK}\n`, "utf8");
+    return;
+  }
+
+  const lines = existing.split(/\r?\n/);
+  const open = lines.indexOf(OPEN_BLOCK);
+  const close = lines.indexOf(CLOSE_BLOCK, open + 1);
+
+  if (open === -1 || close === -1) {
+    // No managed block yet — or a half-written one, which is not something to
+    // guess at. Append a whole one and leave whatever is there alone.
+    writeFileSync(file, `${existing.trimEnd()}\n\n${BLOCK}\n`, "utf8");
+    return;
+  }
+
+  const next = [...lines.slice(0, open), ...BLOCK.split("\n"), ...lines.slice(close + 1)];
+  const body = next.join("\n");
+  writeFileSync(file, body.endsWith("\n") ? body : `${body}\n`, "utf8");
 }

packages/access/src/index.ts

@@ -81,18 +81,34 @@ export {
 } from "./sources/manifest.js";
 export { registerSource, type RegisterInput } from "./sources/register.js";
 export { deriveId, isIdTaken, EmptyNameError } from "./sources/id.js";
+export { uploadTextSource, writeSourceText, normaliseText } from "./sources/ingest.js";
 export {
-  uploadTextSource,
-  writeSourceText,
-  normaliseText,
-} from "./sources/ingest.js";
+  uploadPdfSource,
+  extractPdfPages,
+  renderPdfText,
+  pageAnchor,
+  type PdfPage,
+} from "./sources/pdf.js";
+export { uploadDocxSource, extractDocxMarkdown, htmlToMarkdown } from "./sources/docx.js";
+export {
+  ingestSource,
+  recogniseSource,
+  recognisedExtensions,
+  type SourceFormat,
+  type IngestOutcome,
+} from "./sources/upload.js";
+export {
+  drainInbox,
+  watchInbox,
+  ensureInbox,
+  inboxPath,
+  INBOX,
+  type InboxOutcome,
+  type InboxWatcher,
+  type WatchInboxOptions,
+} from "./sources/inbox.js";
 export { resolveProvenance, extractProvenanceLinks } from "./store/provenance.js";
 export { completeFrontmatter } from "./store/complete.js";
 export { recordWrite, type WriteEntry, type WriteAction } from "./store/record.js";
-export {
-  listEntityPages,
-  isIndexed,
-  findOrphans,
-  readIndex,
-} from "./store/index.js";
+export { listEntityPages, isIndexed, findOrphans, readIndex } from "./store/index.js";
 export { registerInIndex } from "./store/index-write.js";

packages/access/src/scaffold.ts

@@ -3,8 +3,11 @@ import { join } from "node:path";
 import { writeSettings, type ProjectSettings } from "./config/settings.js";
 import { writeIgnore } from "./ignore.js";
 import { scaffoldSkills } from "./skills.js";
+import { INBOX } from "./sources/manifest.js";
 
-const DIRS = ["raw", "wiki", ".state"];
+// `raw/_inbox` is created here rather than on first use: a doorway nobody can
+// see is a doorway nobody drops anything through (plan 3.7).
+const DIRS = ["raw", join("raw", INBOX), "wiki", ".state"];
 
 export class DirectoryOccupiedError extends Error {
   constructor(public readonly dir: string) {