From 50ec3a63766845098d21cae4da5b94a9f56db75d Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sat, 1 Aug 2026 08:20:34 +0000 Subject: [PATCH 1/3] feat(access): ingest PDF, DOCX and the raw/_inbox doorway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes plan tasks 3.3, 3.4 and 3.7. 3.3 — A PDF's text is extracted per page and written under a `## p` heading, so the fragment of `src://#p12` is also the markdown anchor of the heading it points at. A page with no extractable text keeps its anchor: dropping it would shift every page number after it. pdfjs-dist rather than the pdf-parse this repo's stack.md had pencilled in — pdf-parse wraps an old fork of the same engine and returns the document as one string, so the page boundary, the only thing a citation needs, has to be recovered through a render hook. 3.4 — A DOCX's text and heading hierarchy, read with mammoth. Its own markdown writer is deprecated and escapes ordinary prose (`in two\.`), so this converts mammoth's HTML subset itself. No page anchor: the format records no pagination, and a synthetic `p` would be a number that looks like provenance and points nowhere. That leaves a real gap against the shipped 5.4, which accepts only `p` — recorded on the plan's 3.4 line rather than left implied by a comment. 3.7 — `raw/_inbox/` is scaffolded, gitignored and drained through the same path as an upload. It is not a source: nothing enumerates it, cites it, or reports it uncited. A file that cannot be ingested stays where it is with the reason reported — it is the user's only copy. Also from the two reviews on this branch: - The watcher ingests the file an event named, not the whole directory. chokidar's awaitWriteFinish stabilises one path; re-reading everything on that event picked up a neighbour still mid-copy, froze half of it as an immutable source and deleted the original. It also applies no stability check at all before `ready`, so the guarantee is ours: each file is observed twice before it is read. - The doorway is confined against the project, not against `raw/`, and registerSource confines before it creates anything. Rooting the assertion at `raw/` asserted nothing about `raw/` itself, so a symlink there landed bytes outside the project and still reported failure. - Size ceilings on both readers. A 551 KB DOCX declaring 166 MB of XML exhausted the heap, which V8 aborts rather than throws, so no try/catch downstream contained it — and the file survived to kill the next start too. The zip's declared sizes are checked before anything inflates. - The managed .gitignore block is rewritten rather than skipped, so a rule added later reaches a project scaffolded earlier. Opting in moves to a negation below the closing marker, which git honours and the tool never rewrites — something the file can state, unlike an edit inside the block that nothing could tell from a mistake. - chokidar's `error` event is handled. Unhandled, an EventEmitter error throws and takes the host process with it. - Table cells, multi-paragraph list items, nested ordered-list indents and emphasis runs carrying an edge space are all handled; the tests that covered them asserted markup mammoth never emits. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016iMM93Wk43o44V5J2AxPgL --- docs/stack.md | 5 +- packages/access/package.json | 3 + packages/access/src/ignore.ts | 58 ++- packages/access/src/index.ts | 36 +- packages/access/src/scaffold.ts | 5 +- packages/access/src/sources/docx.ts | 367 +++++++++++++++++++ packages/access/src/sources/id.ts | 3 +- packages/access/src/sources/inbox.ts | 336 +++++++++++++++++ packages/access/src/sources/manifest.ts | 14 +- packages/access/src/sources/pdf.ts | 138 +++++++ packages/access/src/sources/register.ts | 27 +- packages/access/src/sources/upload.ts | 125 +++++++ packages/access/tests/fixtures/documents.ts | 182 +++++++++ packages/access/tests/ignore.spec.ts | 57 ++- packages/access/tests/sources-docx.spec.ts | 230 ++++++++++++ packages/access/tests/sources-inbox.spec.ts | 356 ++++++++++++++++++ packages/access/tests/sources-pdf.spec.ts | 149 ++++++++ packages/access/tests/sources-upload.spec.ts | 144 ++++++++ packages/access/tsconfig.json | 2 +- packages/access/types/mammoth.d.ts | 30 ++ plans/open-wiki.md | 10 +- pnpm-lock.yaml | 315 ++++++++++++++++ 22 files changed, 2542 insertions(+), 50 deletions(-) create mode 100644 packages/access/src/sources/docx.ts create mode 100644 packages/access/src/sources/inbox.ts create mode 100644 packages/access/src/sources/pdf.ts create mode 100644 packages/access/src/sources/upload.ts create mode 100644 packages/access/tests/fixtures/documents.ts create mode 100644 packages/access/tests/sources-docx.spec.ts create mode 100644 packages/access/tests/sources-inbox.spec.ts create mode 100644 packages/access/tests/sources-pdf.spec.ts create mode 100644 packages/access/tests/sources-upload.spec.ts create mode 100644 packages/access/types/mammoth.d.ts diff --git a/docs/stack.md b/docs/stack.md index 74f373f..1124bf4 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -42,8 +42,9 @@ Each source adapter has a single responsibility — becoming `text.md` with prov anchors, and the path stops writing there — see `adr:0013-the-project-directory-is-the-unit`. -- **pdf-parse** — text and page boundaries of a PDF; it is the page number that makes the citation possible, and without it the source is of no use. -- **mammoth** — DOCX to markdown preserving the heading hierarchy, which is the anchor equivalent to a PDF's page. +- **pdfjs-dist** — text and page boundaries of a PDF; it is the page number that makes the citation possible, and without it the source is of no use. Chosen over `pdf-parse`, which this file named before the adapter was built: `pdf-parse` wraps an old fork of this same engine and hands back the whole document as one string, so the page boundary — the only thing the citation needs — has to be recovered through a render hook. `pdfjs-dist` has `getPage(n).getTextContent()`, which is the boundary directly, and it is the engine upstream maintains. It declares `@napi-rs/canvas` as an *optional* dependency — a native binary, per platform — which rendering needs and text extraction does not; nothing here imports it, and the installer of 10.1 should not carry it. +- **mammoth** — DOCX to markdown preserving the heading hierarchy, which is what a DOCX carries instead of a PDF's page: the file records no pagination, so the structure is the anchor. Its own markdown writer is deprecated and escapes ordinary prose (`split in two\.`), so the adapter converts mammoth's HTML — a small, predictable subset — itself. +- **chokidar** — watches `raw/_inbox/` for material an agent dropped there (plan 3.7), and later the project folder (8.10). `fs.watch` alone reports a file the moment it appears, which on a copy is halfway through being written; `awaitWriteFinish` is the part that stops a half-copied PDF becoming a permanently wrong source. ## MCP server diff --git a/packages/access/package.json b/packages/access/package.json index 27cfbcf..5dc0813 100644 --- a/packages/access/package.json +++ b/packages/access/package.json @@ -15,6 +15,9 @@ "lint": "eslint ." }, "dependencies": { + "chokidar": "^5.0.0", + "mammoth": "^1.12.0", + "pdfjs-dist": "^6.2.108", "yaml": "^2.7.0" } } diff --git a/packages/access/src/ignore.ts b/packages/access/src/ignore.ts index e7b5159..cbec02d 100644 --- a/packages/access/src/ignore.ts +++ b/packages/access/src/ignore.ts @@ -11,26 +11,64 @@ import { join } from "node:path"; export const OPEN_BLOCK = "# >>> open-wiki >>>"; export const CLOSE_BLOCK = "# <<< open-wiki <<<"; -const BLOCK = [ - OPEN_BLOCK, +const BODY = [ "# Recorded audio and .state/ are ignored by default; committing them is", "# opt-in. .state/ holds every page as it was before each write, which is", "# where a redaction survives the redaction (adr:0013).", ".state/", "raw/**/*.wav", "raw/**/*.opus", - CLOSE_BLOCK, -].join("\n"); + "# raw/_inbox/ is a doorway, emptied by ingestion (plan 3.7). What sits in it", + "# has not been read yet, and committing unreviewed material is not something", + "# to do by default; a file that became a source is committed as that source.", + "raw/_inbox/", + "#", + "# Everything between the two markers is managed: it is rewritten whenever the", + "# project is scaffolded, so a rule added in a later version reaches a project", + "# created by an earlier one. To commit something this ignores, put a negation", + "# *below* the closing marker — git takes the last matching pattern, and a line", + "# outside the block is never touched.", +]; + +const BLOCK = [OPEN_BLOCK, ...BODY, CLOSE_BLOCK].join("\n"); /** - * Writes the managed block into `/.gitignore`, idempotently. A block - * already present is left untouched (the user may have opted in by editing it), - * and any other content is preserved. + * Writes the managed block into `/.gitignore`. Content outside the + * markers is preserved exactly; content between them is replaced. + * + * **Replaced, not skipped.** Leaving an existing block untouched meant a rule + * added later never reached a project scaffolded earlier — and `scaffold` runs + * again on an existing project, so those projects got the new *directory* + * without the new ignore rule. For `raw/_inbox/` that is precisely backwards: + * the doorway appears, and the unreviewed material an agent drops in it is + * git-visible in exactly the projects that already existed. + * + * Opting in is still supported, and is now a thing the file can express rather + * than a thing the tool has to infer from an edit it cannot tell from a + * mistake: a negation below the closing marker wins, because git takes the last + * matching pattern. */ export function writeIgnore(projectRoot: string): void { const file = join(projectRoot, ".gitignore"); const existing = existsSync(file) ? readFileSync(file, "utf8") : ""; - if (existing.includes(OPEN_BLOCK)) return; // already managed — do not clobber - const body = existing.length === 0 ? BLOCK : `${existing.trimEnd()}\n\n${BLOCK}\n`; - writeFileSync(file, body, "utf8"); + + if (existing === "") { + writeFileSync(file, `${BLOCK}\n`, "utf8"); + return; + } + + const lines = existing.split(/\r?\n/); + const open = lines.indexOf(OPEN_BLOCK); + const close = lines.indexOf(CLOSE_BLOCK, open + 1); + + if (open === -1 || close === -1) { + // No managed block yet — or a half-written one, which is not something to + // guess at. Append a whole one and leave whatever is there alone. + writeFileSync(file, `${existing.trimEnd()}\n\n${BLOCK}\n`, "utf8"); + return; + } + + const next = [...lines.slice(0, open), ...BLOCK.split("\n"), ...lines.slice(close + 1)]; + const body = next.join("\n"); + writeFileSync(file, body.endsWith("\n") ? body : `${body}\n`, "utf8"); } diff --git a/packages/access/src/index.ts b/packages/access/src/index.ts index bcb66c0..e4b86b4 100644 --- a/packages/access/src/index.ts +++ b/packages/access/src/index.ts @@ -81,18 +81,34 @@ export { } from "./sources/manifest.js"; export { registerSource, type RegisterInput } from "./sources/register.js"; export { deriveId, isIdTaken, EmptyNameError } from "./sources/id.js"; +export { uploadTextSource, writeSourceText, normaliseText } from "./sources/ingest.js"; export { - uploadTextSource, - writeSourceText, - normaliseText, -} from "./sources/ingest.js"; + uploadPdfSource, + extractPdfPages, + renderPdfText, + pageAnchor, + type PdfPage, +} from "./sources/pdf.js"; +export { uploadDocxSource, extractDocxMarkdown, htmlToMarkdown } from "./sources/docx.js"; +export { + ingestSource, + recogniseSource, + recognisedExtensions, + type SourceFormat, + type IngestOutcome, +} from "./sources/upload.js"; +export { + drainInbox, + watchInbox, + ensureInbox, + inboxPath, + INBOX, + type InboxOutcome, + type InboxWatcher, + type WatchInboxOptions, +} from "./sources/inbox.js"; export { resolveProvenance, extractProvenanceLinks } from "./store/provenance.js"; export { completeFrontmatter } from "./store/complete.js"; export { recordWrite, type WriteEntry, type WriteAction } from "./store/record.js"; -export { - listEntityPages, - isIndexed, - findOrphans, - readIndex, -} from "./store/index.js"; +export { listEntityPages, isIndexed, findOrphans, readIndex } from "./store/index.js"; export { registerInIndex } from "./store/index-write.js"; diff --git a/packages/access/src/scaffold.ts b/packages/access/src/scaffold.ts index 9bc3d7c..ebef4db 100644 --- a/packages/access/src/scaffold.ts +++ b/packages/access/src/scaffold.ts @@ -3,8 +3,11 @@ import { join } from "node:path"; import { writeSettings, type ProjectSettings } from "./config/settings.js"; import { writeIgnore } from "./ignore.js"; import { scaffoldSkills } from "./skills.js"; +import { INBOX } from "./sources/manifest.js"; -const DIRS = ["raw", "wiki", ".state"]; +// `raw/_inbox` is created here rather than on first use: a doorway nobody can +// see is a doorway nobody drops anything through (plan 3.7). +const DIRS = ["raw", join("raw", INBOX), "wiki", ".state"]; export class DirectoryOccupiedError extends Error { constructor(public readonly dir: string) { diff --git a/packages/access/src/sources/docx.ts b/packages/access/src/sources/docx.ts new file mode 100644 index 0000000..7eba3a4 --- /dev/null +++ b/packages/access/src/sources/docx.ts @@ -0,0 +1,367 @@ +import { registerSource } from "./register.js"; +import { writeSourceText } from "./ingest.js"; + +/** + * Upload a DOCX (plan 3.4): extract the text and the heading hierarchy to + * `text.md`. + * + * **The hierarchy is what a DOCX has instead of pages.** A PDF is laid out, so + * a passage has a page number and 3.3 makes that the provenance anchor. A DOCX + * is not: pagination happens when Word renders it, and nothing in the file says + * where page 7 begins. So this path preserves the structure the file *does* + * carry — the heading levels — and writes no page anchor rather than inventing + * one. A synthetic `p` would be a number that looks like provenance and + * points nowhere, which is the failure the plan calls worse than not existing. + * + * mammoth does the reading, because a DOCX is a zip of XML with a style system, + * and the mapping from a named paragraph style to a heading level is the part + * that is not worth re-deriving. Its own markdown writer is deprecated and + * escapes ordinary prose (`split in two\.`), so this converts mammoth's HTML — + * a small, predictable subset — itself. + * + * Loaded through a dynamic import for the same reason as the PDF path: a hook + * fires this package on every page write and must not pay for a DOCX reader. + */ + +/** Decode the entities mammoth emits in text; it escapes nothing else. */ +function decodeEntities(text: string): string { + return ( + text + .replace(/ /g, " ") + .replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code))) + .replace(/&#x([0-9a-f]+);/gi, (_, code: string) => String.fromCodePoint(parseInt(code, 16))) + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, "<") + .replace(/>/g, ">") + // Ampersand last: decoding it first would turn `&lt;` into `<`. + .replace(/&/g, "&") + ); +} + +/** Squeeze the whitespace HTML treats as insignificant, keeping `
` breaks. */ +function collapse(text: string): string { + return text + .split("\n") + .map((line) => line.replace(/[ \t]+/g, " ").trim()) + .join("\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +interface Block { + /** List items pack tight; every other block gets a blank line around it. */ + list: boolean; + text: string; +} + +/** One open list, and the marker width its children have to indent past. */ +interface ListLevel { + ordered: boolean; + n: number; + /** Columns the parent item's content starts at — `- ` is 2, `10. ` is 4. */ + indent: number; +} + +const TAG = /<(\/?)([a-z][a-z0-9]*)((?:\s[^>]*?)?)\/?>/gi; + +/** + * Move an emphasis run's edge whitespace outside its delimiters. + * + * Word records "bold this word and the space after it" constantly, and mammoth + * reports it faithfully as `bold `. Emitting `**bold **` is not + * emphasis in CommonMark — a closing `**` preceded by whitespace is not + * right-flanking — so the reader sees the literal asterisks. The delimiters + * belong around the trimmed text, with the spaces left where they were. + */ +function emphasise(run: string, delimiter: string): string { + const core = run.trim(); + if (core === "") return run; // nothing to emphasise; keep the spacing + const lead = run.slice(0, run.indexOf(core[0]!)); + const trail = run.slice(lead.length + core.length); + return `${lead}${delimiter}${core}${delimiter}${trail}`; +} + +/** + * Convert the HTML subset mammoth emits into markdown, preserving the heading + * hierarchy. Unknown tags are unwrapped rather than dropped, so a construct + * this does not model loses its formatting and never its text. + * + * Nothing here escapes markdown punctuation. Escaping is what made mammoth's + * own writer unusable, and `text.md` is read — by a person and by the agent — + * not round-tripped back into a document. + */ +export function htmlToMarkdown(html: string): string { + const blocks: Block[] = []; + const lists: ListLevel[] = []; + let buffer = ""; + let heading = 0; + let quoted = false; + let href: string | null = null; + // A cell's text is collected, not flushed: `

` inside `` must not end + // the row. `null` means "not inside a table row". + let cells: string[] | null = null; + let inCell = false; + // A list item's own paragraphs belong to that item; only `` ends it. + let inItem = false; + // Where each open emphasis run started in `buffer`, so its delimiters can be + // placed around the trimmed text when it closes. + const runs: Array<{ at: number; delimiter: string }> = []; + + const flush = (): void => { + const text = collapse(buffer); + buffer = ""; + runs.length = 0; + if (text === "") return; + if (heading > 0) { + blocks.push({ list: false, text: `${"#".repeat(heading)} ${text}` }); + return; + } + const top = lists[lists.length - 1]; + if (top) { + const indent = " ".repeat(top.indent); + const marker = top.ordered ? `${top.n}. ` : "- "; + // Continuation lines line up under the item's content, or CommonMark + // reads them as a new block rather than as part of this item. A blank + // line stays blank — padding it would leave trailing whitespace, which + // every linter and diff flags. + const pad = " ".repeat(indent.length + marker.length); + const body = text + .split("\n") + .map((line, i) => (i === 0 || line === "" ? line : pad + line)) + .join("\n"); + blocks.push({ list: true, text: `${indent}${marker}${body}` }); + return; + } + blocks.push({ list: false, text: quoted ? `> ${text.replace(/\n/g, "\n> ")}` : text }); + }; + + /** End a paragraph. Inside a cell or a list item it is a break, not a block. */ + const endParagraph = (): void => { + if (inCell || inItem) { + if (buffer.trim() !== "") buffer += "\n\n"; + return; + } + flush(); + }; + + const openEmphasis = (delimiter: string): void => { + runs.push({ at: buffer.length, delimiter }); + }; + + const closeEmphasis = (delimiter: string): void => { + // Match the innermost open run with this delimiter; an unmatched close is + // dropped rather than emitting a stray `**`. + for (let i = runs.length - 1; i >= 0; i--) { + if (runs[i]!.delimiter !== delimiter) continue; + const { at } = runs[i]!; + runs.splice(i, 1); + buffer = buffer.slice(0, at) + emphasise(buffer.slice(at), delimiter); + return; + } + }; + + let cursor = 0; + for (const match of html.matchAll(TAG)) { + buffer += decodeEntities(html.slice(cursor, match.index)); + cursor = match.index + match[0].length; + + const closing = match[1] === "/"; + const tag = match[2]!.toLowerCase(); + const attrs = match[3] ?? ""; + + const level = /^h([1-6])$/.exec(tag); + if (level) { + flush(); + heading = closing ? 0 : Number(level[1]); + continue; + } + + switch (tag) { + case "p": + endParagraph(); + break; + case "table": + case "thead": + case "tbody": + flush(); + break; + case "tr": + if (closing) { + // The row is one block; an empty cell keeps its column, or every + // cell after it shifts left and the table says something else. + if (cells !== null && cells.length > 0) { + blocks.push({ list: false, text: cells.join(" | ") }); + } + cells = null; + } else { + flush(); + cells = []; + } + break; + case "td": + case "th": + if (closing) { + cells?.push(collapse(buffer).replace(/\n+/g, " ")); + buffer = ""; + runs.length = 0; + inCell = false; + } else { + if (cells === null) cells = []; // a cell outside any row + buffer = ""; + runs.length = 0; + inCell = true; + } + break; + case "ul": + case "ol": + flush(); + if (closing) { + lists.pop(); + } else { + const parent = lists[lists.length - 1]; + // Indent past the parent item's marker, not by a fixed two spaces: + // under `1. ` the content column is 3, and two spaces would make the + // child a sibling instead of a child. + const indent = parent ? parent.indent + (parent.ordered ? `${parent.n}. `.length : 2) : 0; + lists.push({ ordered: tag === "ol", n: 0, indent }); + } + break; + case "li": { + flush(); + inItem = !closing; + const top = lists[lists.length - 1]; + if (!closing && top) top.n += 1; + break; + } + case "blockquote": + flush(); + quoted = !closing; + break; + case "br": + buffer += "\n"; + break; + case "strong": + case "b": + if (closing) closeEmphasis("**"); + else openEmphasis("**"); + break; + case "em": + case "i": + if (closing) closeEmphasis("_"); + else openEmphasis("_"); + break; + case "a": + if (closing) { + buffer += href === null ? "" : `](${href})`; + href = null; + } else { + const url = /\bhref\s*=\s*"([^"]*)"/i.exec(attrs); + href = url ? decodeEntities(url[1]!) : null; + if (href !== null) buffer += "["; + } + break; + case "img": + // A picture carries no text, and its data URI would swamp the page. + break; + default: + break; // unwrap: keep the text, lose the tag + } + } + buffer += decodeEntities(html.slice(cursor)); + flush(); + + let out = ""; + blocks.forEach((block, i) => { + if (i > 0) out += block.list && blocks[i - 1]!.list ? "\n" : "\n\n"; + out += block.text; + }); + return out; +} + +/** + * The most a DOCX may inflate to. A zip entry can expand by a factor of a + * thousand, and mammoth holds the whole of `word/document.xml` as one string + * and then as a DOM — so a 550 KB file that declares 166 MB of XML exhausts the + * heap. That is not a catchable exception: V8 aborts the process, so no + * `try/catch` downstream contains it, and because a file is only removed from + * the inbox on success the same file is re-read on every start. + */ +export const MAX_DOCX_UNCOMPRESSED_BYTES = 64 * 1024 * 1024; + +/** The end-of-central-directory record can sit behind a comment up to 64 KiB. */ +const EOCD_SIGNATURE = 0x06054b50; +const CENTRAL_SIGNATURE = 0x02014b50; +const ZIP64_SENTINEL = 0xffffffff; + +/** + * Sum the uncompressed sizes a zip's central directory **declares**, without + * inflating anything. Returns `null` when the structure cannot be read — a + * malformed archive is mammoth's to report, not this guard's to guess at. + */ +export function declaredUncompressedSize(zip: Buffer): number | null { + const start = Math.max(0, zip.length - (22 + 0xffff)); + let eocd = -1; + for (let i = zip.length - 22; i >= start; i--) { + if (zip.readUInt32LE(i) === EOCD_SIGNATURE) { + eocd = i; + break; + } + } + if (eocd === -1) return null; + + const entries = zip.readUInt16LE(eocd + 10); + let offset = zip.readUInt32LE(eocd + 16); + if (offset === ZIP64_SENTINEL) return Number.POSITIVE_INFINITY; // ZIP64: refuse + + let total = 0; + for (let n = 0; n < entries; n++) { + if (offset + 46 > zip.length) return null; + if (zip.readUInt32LE(offset) !== CENTRAL_SIGNATURE) return null; + const size = zip.readUInt32LE(offset + 24); + // A ZIP64 entry hides its real size in an extra field. Rather than parse + // that, treat it as over any ceiling: a document that needs ZIP64 is not a + // document this reads. + if (size === ZIP64_SENTINEL) return Number.POSITIVE_INFINITY; + total += size; + offset += + 46 + + zip.readUInt16LE(offset + 28) + + zip.readUInt16LE(offset + 30) + + zip.readUInt16LE(offset + 32); + } + return total; +} + +/** + * Read a DOCX and return its body as markdown, headings and all. Refuses a + * document that declares more inflated bytes than the ceiling above, before + * handing anything to the reader. + */ +export async function extractDocxMarkdown(content: Buffer): Promise { + const declared = declaredUncompressedSize(content); + if (declared !== null && declared > MAX_DOCX_UNCOMPRESSED_BYTES) { + throw new Error( + `this DOCX declares ${declared} bytes of content, over the ${MAX_DOCX_UNCOMPRESSED_BYTES}-byte limit — ` + + "a document that large is not one this reads", + ); + } + const mammoth = (await import("mammoth")).default; + const { value } = await mammoth.convertToHtml({ buffer: content }); + return htmlToMarkdown(value); +} + +/** + * Register a DOCX under `raw//` — the original preserved as `source.docx` — + * and write its extracted `text.md`. Returns the frozen id. + */ +export async function uploadDocxSource( + projectRoot: string, + name: string, + content: Buffer, +): Promise<{ id: string }> { + const markdown = await extractDocxMarkdown(content); + const { id } = registerSource(projectRoot, { name, kind: "file", content }); + writeSourceText(projectRoot, id, markdown); + return { id }; +} diff --git a/packages/access/src/sources/id.ts b/packages/access/src/sources/id.ts index 3f6d756..39c7603 100644 --- a/packages/access/src/sources/id.ts +++ b/packages/access/src/sources/id.ts @@ -1,5 +1,6 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; +import { INBOX } from "./manifest.js"; /** * A source's directory name is derived from what the source is, and frozen @@ -37,8 +38,6 @@ export function deriveId(name: string): string { return trimmed + ext; } -const INBOX = "_inbox"; - /** True when a source directory with this id already exists under `raw/`. */ export function isIdTaken(projectRoot: string, id: string): boolean { if (id === INBOX) return false; // the inbox is a doorway, not a source diff --git a/packages/access/src/sources/inbox.ts b/packages/access/src/sources/inbox.ts new file mode 100644 index 0000000..1ef5ce1 --- /dev/null +++ b/packages/access/src/sources/inbox.ts @@ -0,0 +1,336 @@ +import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { basename, join } from "node:path"; +import { assertWithin } from "../paths.js"; +import { INBOX } from "./manifest.js"; +import { MAX_SOURCE_BYTES, ingestSource, type IngestOutcome } from "./upload.js"; + +/** + * `raw/_inbox/` — the doorway (plan 3.7). It is how an agent hands over material + * it fetched, now that no MCP tool ingests: it writes the file into the inbox + * with its own tools and the ingestion happens here, through the same path as a + * file the user dragged onto the window (3.1). + * + * **The inbox is the one mutable thing under `raw/`.** It is not a source: it is + * emptied by ingestion, nothing enumerates it (`listSources` skips it), nothing + * cites it, and 6.6's "sitting in raw/ and never cited" never reports it. What + * a file in the inbox turns into is a source, immutable like any other. + * + * A file that could not be ingested **stays where it is**, with the reason + * reported. It is the user's file, and moving or deleting it to keep the + * doorway tidy would lose material that nothing else holds a copy of. + * + * Nothing wires the watcher up yet — group 8's shell is what will run it. Until + * then `drainInbox` is the whole doorway, callable by hand. + */ + +export { INBOX }; + +/** + * The inbox directory of a project. + * + * Confined against the **project**, not against `raw/`. Rooting the assertion at + * `raw/` asserts nothing about `raw/` itself, so a symlink or Windows junction + * standing where `raw/` should be would carry the whole doorway — and the reads + * and writes that follow it — outside the project. That is the escape + * `resolveReal` exists to stop (`paths.ts`), and it has to be checked at the + * outermost segment the caller controls. + */ +export function inboxPath(projectRoot: string): string { + return assertWithin(projectRoot, join(projectRoot, "raw", INBOX)); +} + +/** Create the inbox if it is not there. Called by the scaffolder and the watcher. */ +export function ensureInbox(projectRoot: string): string { + const dir = inboxPath(projectRoot); + mkdirSync(dir, { recursive: true }); + return dir; +} + +/** What happened to one file in the inbox, and whether it is still there. */ +export type InboxOutcome = IngestOutcome & { removed: boolean }; + +export { MAX_SOURCE_BYTES }; + +/** + * Ingest everything sitting in the inbox, in name order, and remove each file + * that became a source. Returns one outcome per file it looked at. + * + * Idempotent: a second call sees only what the first could not ingest. + * + * **This reads the whole directory.** Pass `stabilityMs` to have each file + * observed twice before it is read, which is what keeps a file still being + * copied in from being frozen as half a source. `watchInbox` sets it, and also + * ingests the one path an event named rather than re-reading everything. + */ +export async function drainInbox( + projectRoot: string, + options: IngestEntryOptions = {}, +): Promise { + const dir = inboxPath(projectRoot); + if (!existsSync(dir)) return []; + + const outcomes: InboxOutcome[] = []; + // Sorted so a batch is reported in a stable order rather than in whatever + // order the filesystem happens to return. + for (const name of readdirSync(dir).sort()) { + // One entry must never end the batch. Everything below reports its own + // reason and moves to the next file — the caller dropped several. + const outcome = await ingestInboxEntry(projectRoot, name, options); + // `null` is "still being written": not a refusal, and not this drain's to + // report. The next event picks it up. + if (outcome !== null) outcomes.push(outcome); + } + return outcomes; +} + +export interface IngestEntryOptions { + /** + * Observe the file twice, this many milliseconds apart, and skip it when it + * changed in between — it is still being written. + * + * This is ours rather than chokidar's on purpose. chokidar applies + * `awaitWriteFinish` **only after it has emitted `ready`** (`index.js`: the + * `_readyEmitted` guard on the add/change branch), so every file present at + * startup, and every file created in the window before the initial scan + * finishes, is announced with no stability check at all. Reading one of those + * mid-copy freezes half a document as an immutable source and deletes the + * user's only copy. A guarantee that holds on every path has to live here. + */ + stabilityMs?: number; +} + +/** True when the file's size and mtime are the same on both sides of the wait. */ +async function hasSettled(file: string, stabilityMs: number): Promise { + const before = lstatSync(file); + await new Promise((resolve) => setTimeout(resolve, stabilityMs)); + if (!existsSync(file)) return false; + const after = lstatSync(file); + return before.size === after.size && before.mtimeMs === after.mtimeMs; +} + +/** + * Ingest one named entry of the inbox, reporting rather than throwing. `name` + * is a bare filename; a path is reduced to its basename, because the inbox is + * flat and nothing below it is the caller's to reach. + * + * Returns `null` when the file is still being written — not a refusal, because + * nothing is wrong with it: it is simply not finished, and the next event picks + * it up. + */ +export async function ingestInboxEntry( + projectRoot: string, + entryName: string, + options: IngestEntryOptions = {}, +): Promise { + const name = basename(entryName); + const refuse = (reason: string): InboxOutcome => ({ + ok: false, + name, + format: null, + reason, + removed: false, + }); + + try { + const dir = inboxPath(projectRoot); + // `lstat` before anything resolves the path: it describes the link itself, + // where `assertWithin` would follow it and report the target. + const stat = lstatSync(join(dir, name)); + + if (stat.isSymbolicLink()) { + // Following it would copy a file from anywhere on disk into the project + // under a name of the dropper's choosing. The inbox takes files, not + // pointers at files — including one pointing back inside the project, + // which would ingest a page as though it were a source. + return refuse(`"${name}" is a symbolic link; the inbox takes files, not links to them`); + } + if (stat.isDirectory()) { + return refuse(`"${name}" is a directory; drop the files themselves into the inbox`); + } + if (!stat.isFile()) { + return refuse(`"${name}" is not a regular file`); + } + if (stat.size > MAX_SOURCE_BYTES) { + // Refused on the stat, so a file too large to hold in memory is never + // read into memory to find that out. + return refuse( + `"${name}" is ${stat.size} bytes, over the ${MAX_SOURCE_BYTES}-byte limit for a source`, + ); + } + + const file = assertWithin(dir, join(dir, name)); + + const stabilityMs = options.stabilityMs ?? 0; + if (stabilityMs > 0 && !(await hasSettled(file, stabilityMs))) return null; + + const outcome = await ingestSource(projectRoot, name, readFileSync(file)); + if (outcome.ok) rmSync(file, { force: true }); + return { ...outcome, removed: outcome.ok }; + } catch (err) { + return refuse(err instanceof Error ? err.message : String(err)); + } +} + +export interface InboxWatcher { + /** Stop watching. Safe to call more than once. */ + close(): Promise; + /** Drain the whole directory now, without waiting for an event. */ + drain(): Promise; +} + +export interface WatchInboxHandlers { + /** Called once per file, whether it became a source or not. */ + onOutcome: (outcome: InboxOutcome) => void; + /** + * Called when the doorway itself stops working — an unreadable directory, a + * watch that failed. Without it these are invisible: the watcher goes quiet + * and a quiet watcher is indistinguishable from an empty inbox. + */ + onError?: (error: Error) => void; +} + +export interface WatchInboxOptions { + /** + * How long a file's size must hold steady before it is considered fully + * written. A file being copied in arrives in pieces, and ingesting half a PDF + * would register a source that is permanently wrong. + */ + stabilityThreshold?: number; + pollInterval?: number; +} + +/** + * Watch the inbox and ingest what lands there. + * + * **Per file, not per directory.** `awaitWriteFinish` holds an event back until + * *that* file's size has settled, so the path an event names is the one that is + * safe to read. Draining the whole directory on every event throws that away: + * the event for a small file that finished copying would pull in a large one + * still mid-copy, register half of it as an immutable source, and delete the + * user's only copy. So an event ingests exactly the file it named. + * + * chokidar is loaded through a dynamic import so the module graph a hook pays + * for stays free of a file watcher. + */ +export async function watchInbox( + projectRoot: string, + handlers: WatchInboxHandlers, + options: WatchInboxOptions = {}, +): Promise { + const dir = ensureInbox(projectRoot); + const { watch } = await import("chokidar"); + + const report = (error: unknown): void => { + handlers.onError?.(error instanceof Error ? error : new Error(String(error))); + }; + + // Work is serialised through this chain: an event and an explicit drain must + // not both read the same file and both try to register the same id. + let queue: Promise = Promise.resolve(); + const enqueue = (work: () => Promise): Promise => { + const next = queue.then(work, work); + queue = next.catch(() => undefined); + return next; + }; + + // A refused file stays in the doorway, so every later drain sees it again and + // the caller — a UI — must not be told twice about the same one. + // + // Keyed by the *file*, not by its name. Remembering the name alone means a + // second, different file dropped under a name that once failed is silently + // suppressed, and silence is how success looks here. Nor can this hang off + // chokidar's `unlink`: its `atomic` option folds a delete-and-recreate into a + // single `change`, so the unlink for exactly this case never arrives. + const refused = new Map(); + + const identify = (name: string): string => { + try { + const stat = lstatSync(join(dir, name)); + return `${stat.ino}:${stat.size}:${stat.mtimeMs}`; + } catch { + return ""; // gone already; the next sighting is a new file + } + }; + + const announce = (outcome: InboxOutcome): void => { + if (outcome.ok) { + refused.delete(outcome.name); + handlers.onOutcome(outcome); + return; + } + const key = `${identify(outcome.name)}|${outcome.reason}`; + if (refused.get(outcome.name) === key) return; + refused.set(outcome.name, key); + handlers.onOutcome(outcome); + }; + + /** Forget refusals whose file has left the doorway, so a re-drop is heard. */ + const forgetDeparted = (): void => { + if (refused.size === 0) return; + let present: Set; + try { + present = new Set(readdirSync(dir)); + } catch { + return; // the directory is unreadable; the drain below reports that + } + for (const name of refused.keys()) { + if (!present.has(name)) refused.delete(name); + } + }; + + const stabilityMs = options.stabilityThreshold ?? 400; + + const drainAll = async (): Promise => { + forgetDeparted(); + const outcomes = await drainInbox(projectRoot, { stabilityMs }); + for (const outcome of outcomes) announce(outcome); + return outcomes; + }; + + const watcher = watch(dir, { + depth: 0, // the doorway is flat + ignoreInitial: false, // whatever is already sitting there is work to do + awaitWriteFinish: { + stabilityThreshold: stabilityMs, + pollInterval: options.pollInterval ?? 100, + }, + }); + + // chokidar's FSWatcher is a node EventEmitter, and an EventEmitter that emits + // `error` with nobody listening *throws* — taking the desktop application or + // the CLI down with it. EMFILE, ELOOP and EIO all reach here. + watcher.on("error", report); + + // Not what makes a re-drop heard — the identity key above is — but it keeps + // the map from growing for the life of the session. + watcher.on("unlink", (path) => refused.delete(basename(path))); + + const consider = (path: string): void => { + void enqueue(async () => { + const outcome = await ingestInboxEntry(projectRoot, path, { stabilityMs }); + if (outcome !== null) announce(outcome); + }).catch(report); + }; + + watcher.on("add", consider); + // A file that was still growing when it was first seen comes back as a + // change. Without this it would sit in the doorway until something else + // happened to trigger a drain. + watcher.on("change", consider); + + // Return only once the initial scan is done. Before `ready` chokidar applies + // no write-stability check of its own, and a caller that starts watching and + // immediately drops a file would otherwise race the scan. + await new Promise((resolve) => { + if (watcher.closed) resolve(); + else watcher.once("ready", () => resolve()); + }); + + return { + drain: () => enqueue(drainAll), + async close() { + await watcher.close(); + await queue.catch(() => undefined); + }, + }; +} diff --git a/packages/access/src/sources/manifest.ts b/packages/access/src/sources/manifest.ts index 27cea55..0824826 100644 --- a/packages/access/src/sources/manifest.ts +++ b/packages/access/src/sources/manifest.ts @@ -11,6 +11,16 @@ import { assertWithin, OutsideProjectError } from "../paths.js"; * the MCP process imports (plan 9.9) pulls no write code. */ +/** + * The doorway under `raw/` (plan 3.7). Its name lives here — the leaf every + * source module already imports — because it is load-bearing in four places at + * once: the scaffolder creates it, `listSources` must skip it, `isIdTaken` must + * not treat it as a taken id, and the inbox itself reads it. Four copies of one + * string mean a rename breaks the quietest of them, and the quietest is + * `listSources` starting to return `_inbox` as a citable source. + */ +export const INBOX = "_inbox"; + export type SourceKind = "file" | "recording"; export interface SourceManifest { @@ -80,11 +90,11 @@ export function listSources(projectRoot: string): string[] { if (!existsSync(raw)) return []; const ids: string[] = []; for (const entry of readdirSync(raw)) { - if (entry === "_inbox") continue; + if (entry === INBOX) continue; const dir = join(raw, entry); if (!statSync(dir).isDirectory()) continue; if (!existsSync(join(dir, "manifest.json"))) continue; ids.push(entry); } return ids; -} \ No newline at end of file +} diff --git a/packages/access/src/sources/pdf.ts b/packages/access/src/sources/pdf.ts new file mode 100644 index 0000000..2189fe9 --- /dev/null +++ b/packages/access/src/sources/pdf.ts @@ -0,0 +1,138 @@ +import { registerSource } from "./register.js"; +import { writeSourceText } from "./ingest.js"; + +/** + * Upload a PDF (plan 3.3): extract the text to `text.md`, keeping the page + * number as a provenance anchor. + * + * **The page number is the point.** A PDF whose text arrives as one undivided + * blob is a source nothing can cite: `resolveProvenance` requires the fragment + * `p`, and there is nothing to resolve it against. So the extraction is + * per-page by construction, and the rendered `text.md` marks each page with a + * `## p` heading — the same token the citation carries, so the fragment of + * `src://#p12` is also the markdown anchor of the heading it points at. + * + * pdfjs-dist is loaded through a dynamic import rather than at module scope: a + * `PreToolUse` hook fires this package's process on every page write, and + * paying for a PDF engine on a path that never opens a PDF is exactly the cold + * start 9.14 is about. + */ + +export interface PdfPage { + /** 1-based, as printed and as cited. */ + page: number; + text: string; +} + +/** + * The most pages this will walk. Each page's text content is held in memory + * until the whole document is rendered, so an unbounded page count is an + * unbounded allocation driven by an untrusted file. + */ +export const MAX_PDF_PAGES = 5000; + +/** The heading that marks where page `n` begins in a rendered `text.md`. */ +export function pageAnchor(page: number): string { + return `## p${page}`; +} + +interface TextItemLike { + str: string; + hasEOL?: boolean; +} + +/** Marked-content items carry no `str`; only text items do. */ +function isTextItem(item: unknown): item is TextItemLike { + return ( + typeof item === "object" && item !== null && typeof (item as TextItemLike).str === "string" + ); +} + +/** + * Join one page's text items back into lines. pdfjs reports a line break as + * `hasEOL` on the item that ends the line; without honouring it every line of + * the page runs into the next one and the text is unreadable. + */ +export function joinTextItems(items: readonly unknown[]): string { + let out = ""; + for (const item of items) { + if (!isTextItem(item)) continue; + out += item.str; + if (item.hasEOL) out += "\n"; + } + // A PDF's line breaks are visual, so runs of blank lines carry no meaning; + // collapse them to a paragraph break and drop the trailing whitespace. + return out + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +/** + * Extract a PDF's text one page at a time. Returns one entry per page, in + * order, including pages whose text is empty — a scanned page still occupies a + * page number, and dropping it would shift every anchor after it. + */ +export async function extractPdfPages(data: Uint8Array): Promise { + const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); + const task = pdfjs.getDocument({ + // pdfjs takes ownership of the buffer it is handed, so give it a copy: the + // caller still holds `data` to write the preserved original with. + data: new Uint8Array(data), + // No system font lookup: this reads text out of the content stream and + // never renders a glyph, so probing the host's fonts is pure latency. + useSystemFonts: false, + // ERRORS only. Text extraction needs no font programme, so the font + // warnings a headless run emits are noise in every caller's log. + verbosity: 0, + }); + try { + // Inside the `try`: a document the reader rejects must still be destroyed, + // and `await task.promise` is exactly where that rejection arrives. Leaving + // it outside skipped the cleanup on the one path that always fails. + const doc = await task.promise; + if (doc.numPages > MAX_PDF_PAGES) { + throw new Error( + `this PDF has ${doc.numPages} pages, over the ${MAX_PDF_PAGES}-page limit — ` + + "a document that large is not one this reads", + ); + } + const pages: PdfPage[] = []; + for (let n = 1; n <= doc.numPages; n++) { + const page = await doc.getPage(n); + const content = await page.getTextContent(); + pages.push({ page: n, text: joinTextItems(content.items) }); + page.cleanup(); + } + return pages; + } finally { + await task.destroy(); + } +} + +/** + * Render extracted pages as the source's `text.md`: each page under its + * `## p` anchor, in order. An empty page keeps its heading — the anchor has + * to exist for the citation to resolve, and "this page has no extractable text" + * is itself worth seeing. + */ +export function renderPdfText(pages: readonly PdfPage[]): string { + return pages + .map(({ page, text }) => (text === "" ? pageAnchor(page) : `${pageAnchor(page)}\n\n${text}`)) + .join("\n\n"); +} + +/** + * Register a PDF under `raw//` — the original preserved as `source.pdf` — + * and write its extracted `text.md`. Returns the frozen id and the page count. + */ +export async function uploadPdfSource( + projectRoot: string, + name: string, + content: Buffer, +): Promise<{ id: string; pages: number }> { + const pages = await extractPdfPages(new Uint8Array(content)); + const { id } = registerSource(projectRoot, { name, kind: "file", content }); + writeSourceText(projectRoot, id, renderPdfText(pages)); + return { id, pages: pages.length }; +} diff --git a/packages/access/src/sources/register.ts b/packages/access/src/sources/register.ts index 834d021..11313cb 100644 --- a/packages/access/src/sources/register.ts +++ b/packages/access/src/sources/register.ts @@ -1,5 +1,6 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { assertWithin } from "../paths.js"; import { deriveId, isIdTaken } from "./id.js"; import { TakenIdError, type SourceKind } from "./manifest.js"; @@ -28,21 +29,27 @@ function originalFile(id: string): string { return `source${ext}`; } -export function registerSource( - projectRoot: string, - input: RegisterInput, -): { id: string } { +export function registerSource(projectRoot: string, input: RegisterInput): { id: string } { const id = deriveId(input.name); if (isIdTaken(projectRoot, id)) throw new TakenIdError(id); - const dir = join(projectRoot, "raw", id); + // Confine before creating anything. `deriveId` cannot produce a separator or + // a `..`, so the id is not the risk — `raw/` standing as a symlink or a + // Windows junction is, and that would put the directory, the preserved + // original and the manifest outside the project. Refusing after the write + // would report a failure with the bytes already on disk. + const dir = assertWithin(projectRoot, join(projectRoot, "raw", id)); + + if (input.kind === "file" && input.content === null) { + // Checked before the directory exists, so a refused registration leaves no + // empty source behind under an id that is now taken forever. + throw new Error(`file source "${id}" has no content to preserve`); + } + mkdirSync(dir, { recursive: true }); if (input.kind === "file") { - if (input.content === null) { - throw new Error(`file source "${id}" has no content to preserve`); - } - writeFileSync(join(dir, originalFile(id)), input.content); + writeFileSync(join(dir, originalFile(id)), input.content!); } const manifest = { @@ -53,4 +60,4 @@ export function registerSource( }; writeFileSync(join(dir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n", "utf8"); return { id }; -} \ No newline at end of file +} diff --git a/packages/access/src/sources/upload.ts b/packages/access/src/sources/upload.ts new file mode 100644 index 0000000..2331289 --- /dev/null +++ b/packages/access/src/sources/upload.ts @@ -0,0 +1,125 @@ +import { basename } from "node:path"; +import { uploadTextSource } from "./ingest.js"; +import { uploadPdfSource } from "./pdf.js"; +import { uploadDocxSource } from "./docx.js"; + +/** + * The largest file any door accepts. The inbox refuses on the `stat`, before + * reading; this is the ceiling for a caller that already holds the bytes — the + * drag-and-drop surface of 3.5, or a test. + */ +export const MAX_SOURCE_BYTES = 64 * 1024 * 1024; + +/** + * One door for every uploaded file, whichever way it arrived — dragged onto the + * window (plan 3.5), dropped into `raw/_inbox/` (3.7), or handed over by the + * CLI. It recognises the format and dispatches to the adapter that turns it + * into `text.md`; registration itself is always 3.1's, so a source is the same + * source whichever door it came through. + * + * Nothing here throws for a condition the person who dropped the file caused — + * an unreadable PDF, a format nobody supports, a name already taken. Those come + * back as an outcome carrying the reason, because the callers are a drag-and-drop + * surface and a watcher, and both have to report a batch rather than stop at the + * first file that did not work. + */ + +export type SourceFormat = "text" | "pdf" | "docx"; + +export type IngestOutcome = + | { ok: true; name: string; format: SourceFormat; id: string } + | { ok: false; name: string; format: SourceFormat | null; reason: string }; + +/** Extension → adapter. Everything absent from here is not recognised. */ +const FORMATS: ReadonlyMap = new Map([ + [".md", "text"], + [".markdown", "text"], + [".txt", "text"], + [".text", "text"], + [".pdf", "pdf"], + [".docx", "docx"], +]); + +/** The lowercased extension of a filename, `.pdf`, or `""` when it has none. */ +function extensionOf(name: string): string { + const base = name.split(/[\\/]/).pop() ?? name; + const dot = base.lastIndexOf("."); + return dot > 0 ? base.slice(dot).toLowerCase() : ""; +} + +/** + * The format this file will be read as, or `null` when nothing reads it. The + * drag-and-drop surface of 3.5 asks this before ingesting, so it can say what + * it recognised and what it did not without touching the disk. + */ +export function recogniseSource(name: string): SourceFormat | null { + return FORMATS.get(extensionOf(name)) ?? null; +} + +/** Every extension a caller may advertise as accepted. */ +export function recognisedExtensions(): string[] { + return [...FORMATS.keys()]; +} + +function unrecognised(name: string): string { + const ext = extensionOf(name); + const known = recognisedExtensions().join(", "); + if (ext === ".doc") { + // The legacy binary format is a different file format wearing a similar + // name; saying "unsupported" without saying that sends people in circles. + return `"${name}" is the legacy .doc format, which nothing here reads — save it as .docx and drop it again`; + } + return ext === "" + ? `"${name}" has no extension, so there is nothing to recognise it by (known: ${known})` + : `"${name}" is a ${ext} file, which is not a source format (known: ${known})`; +} + +/** + * Register an uploaded file and write its `text.md`, choosing the adapter by + * extension. Returns the frozen source id, or the reason it did not land. + */ +export async function ingestSource( + projectRoot: string, + rawName: string, + content: Buffer, +): Promise { + // Reduce to the basename once, here. `recogniseSource` looked past directory + // components while `deriveId` did not, so a caller handing over a full path — + // which the drag-and-drop surface of 3.5 naturally has — was recognised as a + // PDF and then registered under the id `home-u-documents-report.pdf`. One + // answer to "what is `name`" for both. + const name = basename(rawName); + + const format = recogniseSource(name); + if (format === null) return { ok: false, name, format: null, reason: unrecognised(name) }; + + if (content.byteLength > MAX_SOURCE_BYTES) { + return { + ok: false, + name, + format, + reason: `"${name}" is ${content.byteLength} bytes, over the ${MAX_SOURCE_BYTES}-byte limit for a source`, + }; + } + + try { + switch (format) { + case "text": { + const { id } = uploadTextSource(projectRoot, name, content.toString("utf8")); + return { ok: true, name, format, id }; + } + case "pdf": { + const { id } = await uploadPdfSource(projectRoot, name, content); + return { ok: true, name, format, id }; + } + case "docx": { + const { id } = await uploadDocxSource(projectRoot, name, content); + return { ok: true, name, format, id }; + } + } + } catch (err) { + // A taken id, an empty name, a document the reader will not open: all of + // them are things about *this file*, and the caller has more files to try. + return { ok: false, name, format, reason: err instanceof Error ? err.message : String(err) }; + } +} diff --git a/packages/access/tests/fixtures/documents.ts b/packages/access/tests/fixtures/documents.ts new file mode 100644 index 0000000..dddeafc --- /dev/null +++ b/packages/access/tests/fixtures/documents.ts @@ -0,0 +1,182 @@ +import { crc32 } from "node:zlib"; + +/** + * Document fixtures built in-process, so the PDF and DOCX tests assert against + * a document whose content the test states rather than against a binary blob + * checked in beside them. Both writers emit the smallest file the format's + * readers accept. + */ + +/** + * Build a PDF with one page per entry, each entry a list of lines. Objects are + * uncompressed and the cross-reference table is real, so the file is one any + * conforming reader opens — not something that happens to work with pdfjs. + */ +export function buildPdf(pages: readonly (readonly string[])[]): Buffer { + const objects: string[] = []; + const fontObj = 3 + pages.length * 2; + objects[1] = `<< /Type /Catalog /Pages 2 0 R >>`; + const kids = pages.map((_, i) => `${3 + i * 2} 0 R`).join(" "); + objects[2] = `<< /Type /Pages /Kids [${kids}] /Count ${pages.length} >>`; + + pages.forEach((lines, i) => { + const pageNo = 3 + i * 2; + const contentNo = pageNo + 1; + objects[pageNo] = + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents ${contentNo} 0 R ` + + `/Resources << /Font << /F1 ${fontObj} 0 R >> >> >>`; + // Each line is placed 16pt below the last; the vertical move is what makes + // it a new line rather than a continuation of the same one. + const shown = lines + .map((line, n) => `${n === 0 ? "72 720 Td" : "0 -16 Td"} (${escapePdfText(line)}) Tj`) + .join("\n"); + const stream = `BT\n/F1 12 Tf\n${shown}\nET\n`; + objects[contentNo] = `<< /Length ${stream.length} >>\nstream\n${stream}endstream`; + }); + objects[fontObj] = `<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>`; + + let out = "%PDF-1.4\n"; + const offsets: number[] = []; + for (let i = 1; i < objects.length; i++) { + offsets[i] = out.length; + out += `${i} 0 obj\n${objects[i]}\nendobj\n`; + } + const xrefAt = out.length; + out += `xref\n0 ${objects.length}\n0000000000 65535 f \n`; + for (let i = 1; i < objects.length; i++) { + out += `${String(offsets[i]).padStart(10, "0")} 00000 n \n`; + } + out += `trailer\n<< /Size ${objects.length} /Root 1 0 R >>\nstartxref\n${xrefAt}\n%%EOF\n`; + return Buffer.from(out, "latin1"); +} + +/** `(`, `)` and `\` end or escape a PDF string literal. */ +function escapePdfText(text: string): string { + return text.replace(/([\\()])/g, "\\$1"); +} + +/** One paragraph of a DOCX body, optionally carrying a named paragraph style. */ +export interface DocxParagraph { + text: string; + /** `Heading1` … `Heading6`; absent for body text. */ + style?: string; +} + +/** + * Build a DOCX carrying the given paragraphs. A DOCX is an OPC zip: the three + * parts below are the minimum a reader needs to find and read the body. + */ +export function buildDocx(paragraphs: readonly DocxParagraph[]): Buffer { + const body = paragraphs + .map( + ({ text, style }) => + `${style ? `` : ""}` + + `${escapeXml(text)}`, + ) + .join(""); + + return storedZip([ + ["[Content_Types].xml", CONTENT_TYPES], + ["_rels/.rels", RELS], + [ + "word/document.xml", + ` +${body}`, + ], + ]); +} + +const CONTENT_TYPES = ` + + + + +`; + +const RELS = ` + + +`; + +/** + * Build a DOCX whose central directory **declares** `declaredBytes` of + * uncompressed content while holding almost none — the shape of a zip bomb, + * which is what the size guard reads before it inflates anything. Lying in the + * declaration is the point: a fixture that really held 64 MB would test the + * heap rather than the guard. + */ +export function buildBombDocx(declaredBytes: number): Buffer { + const body = ` +`; + return storedZip([ + ["[Content_Types].xml", CONTENT_TYPES], + ["_rels/.rels", RELS], + ["word/document.xml", body, declaredBytes], + ]); +} + +function escapeXml(text: string): string { + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +/** A zip entry: its name, its bytes, and optionally a size it lies about. */ +type ZipEntry = readonly [name: string, text: string, declaredSize?: number]; + +/** + * Write a ZIP with every entry stored uncompressed. Deflate would buy nothing + * on a fixture and would mean pulling a compression library in to build one. + * + * A third element overrides the *declared* uncompressed size without changing + * the bytes, which is how the bomb fixture is built. + */ +function storedZip(entries: readonly ZipEntry[]): Buffer { + const parts: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + + for (const [name, text, declaredSize] of entries) { + const nameBuf = Buffer.from(name, "utf8"); + const data = Buffer.from(text, "utf8"); + const sum = crc32(data); + const declared = declaredSize ?? data.length; + + const local = Buffer.alloc(30 + nameBuf.length); + local.writeUInt32LE(0x04034b50, 0); // local file header + local.writeUInt16LE(20, 4); // version needed + local.writeUInt16LE(0, 8); // method: stored + local.writeUInt32LE(sum, 14); + local.writeUInt32LE(data.length, 18); // compressed size + local.writeUInt32LE(data.length, 22); // uncompressed size + local.writeUInt16LE(nameBuf.length, 26); + nameBuf.copy(local, 30); + parts.push(local, data); + + const cd = Buffer.alloc(46 + nameBuf.length); + cd.writeUInt32LE(0x02014b50, 0); // central directory header + cd.writeUInt16LE(20, 4); // version made by + cd.writeUInt16LE(20, 6); // version needed + cd.writeUInt16LE(0, 10); // method: stored + cd.writeUInt32LE(sum, 16); + cd.writeUInt32LE(data.length, 20); + cd.writeUInt32LE(declared, 24); // uncompressed size — what the guard reads + cd.writeUInt16LE(nameBuf.length, 28); + cd.writeUInt32LE(offset, 42); // offset of the local header + nameBuf.copy(cd, 46); + central.push(cd); + + offset += local.length + data.length; + } + + const cdBuf = Buffer.concat(central); + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); // end of central directory + eocd.writeUInt16LE(entries.length, 8); + eocd.writeUInt16LE(entries.length, 10); + eocd.writeUInt32LE(cdBuf.length, 12); + eocd.writeUInt32LE(offset, 16); + return Buffer.concat([...parts, cdBuf, eocd]); +} diff --git a/packages/access/tests/ignore.spec.ts b/packages/access/tests/ignore.spec.ts index c9176cf..0dce091 100644 --- a/packages/access/tests/ignore.spec.ts +++ b/packages/access/tests/ignore.spec.ts @@ -39,15 +39,56 @@ describe("ignore entries (2.8)", () => { expect(body).toContain(".state/"); }); - it("does not clobber a block the user has edited (still recognised as present)", () => { + it("ignores raw/_inbox/, which holds material nobody has read yet", () => { writeIgnore(root); - // Simulate the user opting in to committing opus by editing inside the block. - const body = readFileSync(join(root, ".gitignore"), "utf8"); - const edited = body.replace("raw/**/*.opus\n", ""); - writeFileSync(join(root, ".gitignore"), edited); - writeIgnore(root); // should not re-add opus or duplicate + expect(readFileSync(join(root, ".gitignore"), "utf8")).toContain("raw/_inbox/"); + }); + + it("brings a project scaffolded by an earlier version up to date", () => { + // The block used to be skipped whenever it was already present, so a rule + // added later never reached a project created earlier — while `scaffold` + // still created the new *directory* in it. For `raw/_inbox/` that is + // exactly backwards: the doorway appears, and the unreviewed material an + // agent drops in it is committed by anyone running `git add -A`. + const old = [OPEN_BLOCK, ".state/", "raw/**/*.wav", CLOSE_BLOCK].join("\n"); + writeFileSync(join(root, ".gitignore"), `node_modules/\n\n${old}\n`); + + writeIgnore(root); + const after = readFileSync(join(root, ".gitignore"), "utf8"); - expect(after.split(OPEN_BLOCK).length).toBe(2); - expect(after).not.toContain("raw/**/*.opus"); + expect(after).toContain("raw/_inbox/"); + expect(after.split(OPEN_BLOCK).length).toBe(2); // still exactly one block + expect(after.startsWith("node_modules/\n")).toBe(true); // the user's lines survive + }); + + it("leaves everything outside the markers alone, above and below", () => { + const above = "node_modules/\n"; + const below = "!raw/_inbox/keep-this.md\n"; + writeIgnore(root); + writeFileSync( + join(root, ".gitignore"), + `${above}${readFileSync(join(root, ".gitignore"), "utf8")}${below}`, + ); + + writeIgnore(root); + + const body = readFileSync(join(root, ".gitignore"), "utf8"); + expect(body.startsWith(above)).toBe(true); + // A negation below the block is how opting in is expressed now: git takes + // the last matching pattern, and a line outside the markers is never + // rewritten. That is a thing the file can state, unlike an edit inside the + // block, which the tool could not tell from a mistake. + expect(body.trimEnd().endsWith("!raw/_inbox/keep-this.md")).toBe(true); + expect(body.split(OPEN_BLOCK).length).toBe(2); + }); + + it("appends a whole block when only half a marker is present", () => { + // A truncated block is not something to guess at: append a real one and + // leave the damaged text where it is for the user to see. + writeFileSync(join(root, ".gitignore"), `${OPEN_BLOCK}\n.state/\n`); + writeIgnore(root); + const body = readFileSync(join(root, ".gitignore"), "utf8"); + expect(body).toContain(CLOSE_BLOCK); + expect(body).toContain("raw/_inbox/"); }); }); diff --git a/packages/access/tests/sources-docx.spec.ts b/packages/access/tests/sources-docx.spec.ts new file mode 100644 index 0000000..4a70a3e --- /dev/null +++ b/packages/access/tests/sources-docx.spec.ts @@ -0,0 +1,230 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + MAX_DOCX_UNCOMPRESSED_BYTES, + declaredUncompressedSize, + extractDocxMarkdown, + htmlToMarkdown, + uploadDocxSource, +} from "../src/sources/docx.js"; +import { readManifest } from "../src/sources/manifest.js"; +import { buildBombDocx, buildDocx } from "./fixtures/documents.js"; + +function tempProject(): string { + const root = mkdtempSync(join(tmpdir(), "ow-docx-")); + mkdirSync(join(root, "raw"), { recursive: true }); + return root; +} + +describe("DOCX upload (3.4)", () => { + let root: string; + beforeEach(() => (root = tempProject())); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + describe("htmlToMarkdown", () => { + it("maps every heading level to its own depth of hash", () => { + const md = htmlToMarkdown("

One

Two

Three

Six
"); + expect(md).toBe("# One\n\n## Two\n\n### Three\n\n###### Six"); + }); + + it("separates paragraphs with a blank line", () => { + expect(htmlToMarkdown("

first

second

")).toBe("first\n\nsecond"); + }); + + it("keeps a heading and the text under it in order", () => { + expect(htmlToMarkdown("

Storage

Postgres.

")).toBe("## Storage\n\nPostgres."); + }); + + it("renders an unordered list as tight bullets", () => { + expect(htmlToMarkdown("
  • a
  • b
")).toBe("- a\n- b"); + }); + + it("numbers an ordered list from one", () => { + expect(htmlToMarkdown("
  1. a
  2. b
  3. c
")).toBe("1. a\n2. b\n3. c"); + }); + + it("indents a nested list under its parent item", () => { + expect(htmlToMarkdown("
  • outer
    • inner
")).toBe( + "- outer\n - inner", + ); + }); + + it("indents a nested ordered list past its parent's marker, not by a fixed two", () => { + // Under `1. ` the content column is 3. Two spaces would make the child a + // sibling of its parent, which is the nesting the requirement asks for + // being silently lost. + expect(htmlToMarkdown("
  1. a
    1. x
")).toBe("1. a\n 1. x"); + }); + + it("keeps a multi-paragraph list item as one item", () => { + // mammoth emits this whenever a Word list item holds more than one + // paragraph; splitting it reports content the document does not contain. + expect(htmlToMarkdown("
  • a

    second para

")).toBe("- a\n\n second para"); + }); + + it("carries bold and italic through as markdown emphasis", () => { + expect(htmlToMarkdown("

a bold and soft word

")).toBe( + "a **bold** and _soft_ word", + ); + }); + + it("moves an emphasis run's edge space outside the delimiters", () => { + // Word records "bold the word and the space after it" constantly, and + // `**bold **tail` is not emphasis in CommonMark — the reader would see + // the asterisks themselves. + expect(htmlToMarkdown("

bold tail

")).toBe("**bold** tail"); + expect(htmlToMarkdown("

lead inside

")).toBe("lead **in**side"); + expect(htmlToMarkdown("

a soft b

")).toBe("a _soft_ b"); + }); + + it("keeps the spacing of an emphasis run that is only whitespace", () => { + expect(htmlToMarkdown("

a b

")).toBe("a b"); + }); + + it("keeps a link's text and its target", () => { + expect(htmlToMarkdown('

see the note

')).toBe( + "see [the note](https://x.test/a)", + ); + }); + + it("decodes entities, and decodes an escaped ampersand only once", () => { + // `&lt;` is a literal "<", not a "<": decoding the ampersand first + // would produce the wrong character. + expect(htmlToMarkdown("

Tom & Jerry <one> &lt;

")).toBe( + "Tom & Jerry <", + ); + }); + + it("breaks a line where the document broke it", () => { + expect(htmlToMarkdown("

one
two

")).toBe("one\ntwo"); + }); + + it("prefixes a quoted block, including its continuation lines", () => { + expect(htmlToMarkdown("

a
b

")).toBe("> a\n> b"); + }); + + it("keeps the text of a tag it does not model, and loses only the tag", () => { + expect(htmlToMarkdown("

a note here

")).toBe("a note here"); + }); + + it("drops an image, which carries no text", () => { + expect(htmlToMarkdown('

beforeafter

')).toBe( + "beforeafter", + ); + }); + + it("renders a table row as its cells, in the shape mammoth actually emits", () => { + // mammoth wraps every cell's text in a paragraph. Asserting against bare + // a passed on a path production never takes, while the real one + // dropped the row structure entirely. + expect( + htmlToMarkdown( + "" + + "

a

b

c

d

", + ), + ).toBe("a | b\n\nc | d"); + }); + + it("keeps an empty cell, so the columns after it do not shift left", () => { + expect(htmlToMarkdown("

b

")).toBe( + " | b", + ); + expect(htmlToMarkdown("

c

")).toBe( + "c | ", + ); + }); + + it("returns nothing for an empty document", () => { + expect(htmlToMarkdown("")).toBe(""); + }); + }); + + describe("extractDocxMarkdown", () => { + it("preserves the heading hierarchy of the document", async () => { + const docx = buildDocx([ + { text: "Fenix architecture", style: "Heading1" }, + { text: "The service is split in two.", style: undefined }, + { text: "Storage", style: "Heading2" }, + { text: "Postgres, one schema per tenant.", style: undefined }, + ]); + const md = await extractDocxMarkdown(docx); + expect(md).toBe( + "# Fenix architecture\n\nThe service is split in two.\n\n" + + "## Storage\n\nPostgres, one schema per tenant.", + ); + }); + + it("leaves ordinary prose unescaped", async () => { + // The deprecated writer emits `two\.`; a backslash in stored text is a + // defect the reader sees, so this path must not produce one. + const md = await extractDocxMarkdown(buildDocx([{ text: "Split in two." }])); + expect(md).toBe("Split in two."); + expect(md).not.toContain("\\"); + }); + + it("starts at the document's own first heading, with nothing prepended", async () => { + // A DOCX has no pages, so nothing synthesises an anchor above the body. + // Asserting the absence of `## p` proved nothing — no path could emit + // one — so assert what is actually there instead. + const md = await extractDocxMarkdown( + buildDocx([{ text: "Requisitos", style: "Heading1" }, { text: "Body" }]), + ); + expect(md.startsWith("# Requisitos")).toBe(true); + }); + + it("refuses a document that declares more content than it will inflate", async () => { + await expect( + extractDocxMarkdown(buildBombDocx(MAX_DOCX_UNCOMPRESSED_BYTES + 1)), + ).rejects.toThrow(/over the \d+-byte limit/); + }); + }); + + describe("declaredUncompressedSize", () => { + it("sums what the central directory declares, without inflating anything", () => { + const docx = buildDocx([{ text: "Body" }]); + const declared = declaredUncompressedSize(docx); + expect(declared).toBeGreaterThan(0); + // Everything is stored uncompressed here, so the declaration is the truth + // and is necessarily smaller than the container. + expect(declared!).toBeLessThan(docx.length); + }); + + it("reads the size a bomb declares rather than the bytes it holds", () => { + const bomb = buildBombDocx(200 * 1024 * 1024); + expect(bomb.length).toBeLessThan(4096); + expect(declaredUncompressedSize(bomb)).toBeGreaterThan(200 * 1024 * 1024 - 1); + }); + + it("returns null for something that is not a zip, leaving the reader to say so", () => { + expect(declaredUncompressedSize(Buffer.from("not a zip at all"))).toBeNull(); + }); + }); + + describe("uploadDocxSource", () => { + it("preserves the original and writes the extracted text.md", async () => { + const docx = buildDocx([ + { text: "Requisitos", style: "Heading1" }, + { text: "O sistema deve responder em 200ms." }, + ]); + const { id } = await uploadDocxSource(root, "Requisitos Fenix.docx", docx); + + expect(id).toBe("requisitos-fenix.docx"); + const dir = join(root, "raw", id); + expect(existsSync(join(dir, "source.docx"))).toBe(true); + expect(readFileSync(join(dir, "source.docx")).equals(docx)).toBe(true); + + const text = readFileSync(join(dir, "text.md"), "utf8"); + expect(text).toBe("# Requisitos\n\nO sistema deve responder em 200ms.\n"); + + expect(readManifest(root, id).original).toBe("Requisitos Fenix.docx"); + }); + + it("refuses a filename already taken rather than inventing a suffix", async () => { + const docx = buildDocx([{ text: "x" }]); + await uploadDocxSource(root, "spec.docx", docx); + await expect(uploadDocxSource(root, "spec.docx", docx)).rejects.toThrow(/already exists/); + }); + }); +}); diff --git a/packages/access/tests/sources-inbox.spec.ts b/packages/access/tests/sources-inbox.spec.ts new file mode 100644 index 0000000..136560b --- /dev/null +++ b/packages/access/tests/sources-inbox.spec.ts @@ -0,0 +1,356 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { + closeSync, + existsSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, + symlinkSync, + truncateSync, + writeFileSync, + writeSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + INBOX, + drainInbox, + ensureInbox, + inboxPath, + watchInbox, + MAX_SOURCE_BYTES, + type InboxOutcome, +} from "../src/sources/inbox.js"; +import { listSources } from "../src/sources/manifest.js"; +import { scaffold } from "../src/scaffold.js"; +import { buildPdf } from "./fixtures/documents.js"; + +function tempProject(): string { + const root = mkdtempSync(join(tmpdir(), "ow-inbox-")); + mkdirSync(join(root, "raw", INBOX), { recursive: true }); + return root; +} + +/** Poll until `predicate` holds, so the watcher tests wait on the event, not a sleep. */ +async function until(predicate: () => boolean, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() > deadline) throw new Error("timed out waiting for the watcher"); + await new Promise((r) => setTimeout(r, 25)); + } +} + +describe("the raw/_inbox doorway (3.7)", () => { + let root: string; + beforeEach(() => (root = tempProject())); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + describe("inboxPath / ensureInbox", () => { + it("is raw/_inbox inside the project", () => { + expect(inboxPath(root)).toBe(join(root, "raw", INBOX)); + }); + + it("creates the directory when it is not there", () => { + rmSync(join(root, "raw", INBOX), { recursive: true, force: true }); + ensureInbox(root); + expect(existsSync(join(root, "raw", INBOX))).toBe(true); + }); + + it("is scaffolded with the project, so the doorway is visible from the start", () => { + const fresh = join(root, "fresh"); + scaffold(fresh); + expect(existsSync(join(fresh, "raw", INBOX))).toBe(true); + }); + }); + + describe("drainInbox", () => { + it("ingests what landed there through the same path as an upload", async () => { + writeFileSync(join(root, "raw", INBOX, "fetched.md"), "# Fetched\n"); + const outcomes = await drainInbox(root); + + expect(outcomes).toHaveLength(1); + expect(outcomes[0]).toMatchObject({ ok: true, id: "fetched.md", removed: true }); + expect(readFileSync(join(root, "raw", "fetched.md", "text.md"), "utf8")).toBe("# Fetched\n"); + }); + + it("empties the doorway of what became a source", async () => { + writeFileSync(join(root, "raw", INBOX, "a.md"), "a"); + await drainInbox(root); + expect(existsSync(join(root, "raw", INBOX, "a.md"))).toBe(false); + }); + + it("ingests a PDF dropped in, with its page anchors", async () => { + writeFileSync(join(root, "raw", INBOX, "paper.pdf"), buildPdf([["page one"]])); + const outcomes = await drainInbox(root); + expect(outcomes[0]).toMatchObject({ ok: true, format: "pdf", removed: true }); + expect(readFileSync(join(root, "raw", "paper.pdf", "text.md"), "utf8")).toContain("## p1"); + }); + + it("reports every file in a batch, in a stable order", async () => { + writeFileSync(join(root, "raw", INBOX, "b.md"), "b"); + writeFileSync(join(root, "raw", INBOX, "a.md"), "a"); + writeFileSync(join(root, "raw", INBOX, "c.mp3"), "c"); + const outcomes = await drainInbox(root); + expect(outcomes.map((o) => o.name)).toEqual(["a.md", "b.md", "c.mp3"]); + }); + + it("leaves a file it could not ingest where it is, with the reason", async () => { + writeFileSync(join(root, "raw", INBOX, "clip.mp3"), "not a source"); + const outcomes = await drainInbox(root); + + expect(outcomes[0]!.ok).toBe(false); + expect(outcomes[0]!.removed).toBe(false); + // The user's file is the only copy of it; tidying the doorway must not + // be a way to lose material. + expect(existsSync(join(root, "raw", INBOX, "clip.mp3"))).toBe(true); + expect((outcomes[0] as { reason: string }).reason).toContain(".mp3"); + }); + + it("refuses to follow a symbolic link out of the project", async () => { + const outside = join(root, "secret.md"); + writeFileSync(outside, "# not the agent's to publish\n"); + symlinkSync(outside, join(root, "raw", INBOX, "innocuous.md")); + + const outcomes = await drainInbox(root); + expect(outcomes[0]!.ok).toBe(false); + expect((outcomes[0] as { reason: string }).reason).toContain("symbolic link"); + expect(listSources(root)).toEqual([]); + }); + + it("keeps draining the rest of the batch after a file it refused", async () => { + // One bad entry ending the whole drain would mean a single dropped + // symlink silently stops every later file from ever being ingested. + symlinkSync(join(root, "raw"), join(root, "raw", INBOX, "a-link.md")); + writeFileSync(join(root, "raw", INBOX, "z-real.md"), "# Real\n"); + + const outcomes = await drainInbox(root); + expect(outcomes.map((o) => o.name)).toEqual(["a-link.md", "z-real.md"]); + expect(outcomes[1]).toMatchObject({ ok: true, id: "z-real.md" }); + }); + + it("refuses a directory rather than walking into it", async () => { + mkdirSync(join(root, "raw", INBOX, "a-folder")); + const outcomes = await drainInbox(root); + expect(outcomes[0]!.ok).toBe(false); + expect((outcomes[0] as { reason: string }).reason).toContain("directory"); + }); + + it("is idempotent — a second drain sees only what the first could not take", async () => { + writeFileSync(join(root, "raw", INBOX, "good.md"), "g"); + writeFileSync(join(root, "raw", INBOX, "bad.mp3"), "b"); + await drainInbox(root); + + const second = await drainInbox(root); + expect(second.map((o) => o.name)).toEqual(["bad.mp3"]); + }); + + it("refuses an oversized file on its stat, without reading it", async () => { + // A file too large to hold in memory must not be read into memory to + // discover that it is too large. + const big = join(root, "raw", INBOX, "huge.pdf"); + writeFileSync(big, Buffer.alloc(1)); + truncateSync(big, MAX_SOURCE_BYTES + 1); + + const outcomes = await drainInbox(root); + expect(outcomes[0]!.ok).toBe(false); + expect((outcomes[0] as { reason: string }).reason).toMatch(/over the \d+-byte limit/); + expect(existsSync(big)).toBe(true); + }); + + it("returns nothing when there is no inbox at all", async () => { + rmSync(join(root, "raw", INBOX), { recursive: true, force: true }); + expect(await drainInbox(root)).toEqual([]); + }); + + it("does not make the inbox itself a source", async () => { + writeFileSync(join(root, "raw", INBOX, "x.md"), "x"); + await drainInbox(root); + // Nothing enumerates the doorway, so it never appears as something to + // cite or as an uncited source. + expect(listSources(root)).toEqual(["x.md"]); + }); + }); + + describe("watchInbox", () => { + // These wait on real filesystem events, not on a fake clock. + const WATCH_TIMEOUT = 20_000; + + /** Start a watcher with short timings, collecting outcomes and errors. */ + async function start(project = root) { + const seen: InboxOutcome[] = []; + const errors: Error[] = []; + const watcher = await watchInbox( + project, + { onOutcome: (o) => seen.push(o), onError: (e) => errors.push(e) }, + { stabilityThreshold: 150, pollInterval: 25 }, + ); + return { seen, errors, watcher }; + } + + it("ingests a file that lands after the watch started", async () => { + const { seen, watcher } = await start(); + try { + writeFileSync(join(root, "raw", INBOX, "late.md"), "# Late\n"); + await until(() => seen.length > 0); + expect(seen[0]).toMatchObject({ ok: true, id: "late.md" }); + expect(existsSync(join(root, "raw", "late.md", "text.md"))).toBe(true); + } finally { + await watcher.close(); + } + }); + + it("picks up what was already sitting there when it started", async () => { + writeFileSync(join(root, "raw", INBOX, "early.md"), "# Early\n"); + const { seen, watcher } = await start(); + try { + await until(() => seen.length > 0); + expect(seen[0]).toMatchObject({ ok: true, id: "early.md" }); + } finally { + await watcher.close(); + } + }); + + it( + "does not ingest a file that is still being written", + { timeout: WATCH_TIMEOUT }, + async () => { + // The stability threshold holds back the event for the file being + // copied — but only for *that* file. Draining the whole directory on + // another file's event reads this one mid-copy, freezes half of it as an + // immutable source, and deletes the user's only copy. + const { seen, watcher } = await start(); + const partial = join(root, "raw", INBOX, "big.md"); + const fd = openSync(partial, "w"); + let copying = true; + // A real copy keeps growing, which is exactly what stops the watcher + // calling it finished. Simulate that rather than writing once and hoping + // the threshold has not elapsed. + const grow = setInterval(() => { + if (copying) writeSync(fd, "chunk\n"); + }, 40); + + try { + // A second file finishes copying and fires its own event. + writeFileSync(join(root, "raw", INBOX, "small.md"), "# Small\n"); + await until(() => seen.some((o) => o.name === "small.md"), 8000); + + // The file still being written must not have been touched. + expect(listSources(root)).toEqual(["small.md"]); + expect(existsSync(partial)).toBe(true); + expect(seen.some((o) => o.name === "big.md")).toBe(false); + + copying = false; + clearInterval(grow); + writeSync(fd, "last\n"); + closeSync(fd); + + await until(() => seen.some((o) => o.name === "big.md"), 8000); + const text = readFileSync(join(root, "raw", "big.md", "text.md"), "utf8"); + expect(text.endsWith("last\n")).toBe(true); + } finally { + copying = false; + clearInterval(grow); + try { + closeSync(fd); + } catch { + // already closed on the happy path + } + await watcher.close(); + } + }, + ); + + it( + "serialises overlapping drains, so nothing loses a race to itself", + { timeout: WATCH_TIMEOUT }, + async () => { + // Asserting only on the successes cannot fail: the loser of a race comes + // back as a refusal (TakenIdError, or ENOENT on a file the winner already + // removed) and is simply filtered out. The refusals are the evidence. + writeFileSync(join(root, "raw", INBOX, "a.md"), "a"); + writeFileSync(join(root, "raw", INBOX, "b.md"), "b"); + writeFileSync(join(root, "raw", INBOX, "c.md"), "c"); + + const { seen, watcher } = await start(); + try { + const batches = await Promise.all([ + watcher.drain(), + watcher.drain(), + watcher.drain(), + watcher.drain(), + ]); + const all = [...batches.flat(), ...seen]; + expect(all.filter((o) => !o.ok)).toEqual([]); + expect(listSources(root).sort()).toEqual(["a.md", "b.md", "c.md"]); + } finally { + await watcher.close(); + } + }, + ); + + it("reports a file it cannot ingest once, not on every later event", async () => { + writeFileSync(join(root, "raw", INBOX, "clip.mp3"), "x"); + const { seen, watcher } = await start(); + try { + await until(() => seen.length > 0); + // The refused file stays in the doorway, so every later drain sees it + // again; the caller is a UI and must not be told twice. + await watcher.drain(); + await watcher.drain(); + expect(seen.filter((o) => o.name === "clip.mp3")).toHaveLength(1); + } finally { + await watcher.close(); + } + }); + + it( + "reports a refused name again once that file has left and come back", + { timeout: WATCH_TIMEOUT }, + async () => { + // Silence is how success looks here. Remembering the name forever means a + // second, different file dropped under a name that once failed is never + // mentioned, and the user believes it landed. + writeFileSync(join(root, "raw", INBOX, "clip.mp3"), "x"); + const { seen, watcher } = await start(); + try { + await until(() => seen.length > 0); + rmSync(join(root, "raw", INBOX, "clip.mp3")); + + writeFileSync(join(root, "raw", INBOX, "clip.mp3"), "a different recording"); + await until(() => seen.filter((o) => o.name === "clip.mp3").length === 2, 8000); + } finally { + await watcher.close(); + } + }, + ); + + it("surfaces a failure of the doorway itself rather than going quiet", async () => { + const { watcher } = await start(); + try { + // Replace raw/ with a link out of the project: the inbox no longer + // resolves inside it, and an explicit drain has to say so. + rmSync(join(root, "raw"), { recursive: true, force: true }); + const elsewhere = mkdtempSync(join(tmpdir(), "ow-elsewhere-")); + try { + symlinkSync(elsewhere, join(root, "raw")); + await expect(watcher.drain()).rejects.toThrow(/outside the project/); + } finally { + rmSync(elsewhere, { recursive: true, force: true }); + } + } finally { + await watcher.close(); + } + }); + + it("stops ingesting once closed", async () => { + const { seen, watcher } = await start(); + await watcher.close(); + writeFileSync(join(root, "raw", INBOX, "after.md"), "a"); + await new Promise((r) => setTimeout(r, 400)); + expect(seen).toHaveLength(0); + expect(existsSync(join(root, "raw", INBOX, "after.md"))).toBe(true); + }); + }); +}); diff --git a/packages/access/tests/sources-pdf.spec.ts b/packages/access/tests/sources-pdf.spec.ts new file mode 100644 index 0000000..63582c3 --- /dev/null +++ b/packages/access/tests/sources-pdf.spec.ts @@ -0,0 +1,149 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + extractPdfPages, + joinTextItems, + pageAnchor, + renderPdfText, + uploadPdfSource, +} from "../src/sources/pdf.js"; +import { readManifest } from "../src/sources/manifest.js"; +import { buildPdf } from "./fixtures/documents.js"; + +function tempProject(): string { + const root = mkdtempSync(join(tmpdir(), "ow-pdf-")); + mkdirSync(join(root, "raw"), { recursive: true }); + return root; +} + +describe("PDF upload (3.3)", () => { + let root: string; + beforeEach(() => (root = tempProject())); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + describe("pageAnchor", () => { + it("is the heading whose markdown anchor equals the citation fragment", () => { + // A citation reads src://#p12; the anchor of `## p12` is #p12, so the + // link the agent writes also points at the right place inside text.md. + expect(pageAnchor(12)).toBe("## p12"); + expect(pageAnchor(1)).toBe("## p1"); + }); + }); + + describe("joinTextItems", () => { + it("breaks a line where the item says the line ended", () => { + const joined = joinTextItems([ + { str: "first line", hasEOL: true }, + { str: "second line", hasEOL: true }, + ]); + expect(joined).toBe("first line\nsecond line"); + }); + + it("keeps items on one line when no item ends it", () => { + expect(joinTextItems([{ str: "one " }, { str: "line" }])).toBe("one line"); + }); + + it("ignores marked-content items, which carry no text", () => { + expect(joinTextItems([{ type: "beginMarkedContent" }, { str: "text" }])).toBe("text"); + }); + + it("collapses runs of blank lines, which are visual and mean nothing", () => { + const joined = joinTextItems([ + { str: "a", hasEOL: true }, + { str: "", hasEOL: true }, + { str: "", hasEOL: true }, + { str: "b", hasEOL: true }, + ]); + expect(joined).toBe("a\n\nb"); + }); + }); + + describe("extractPdfPages", () => { + it("returns one entry per page, in order, with that page's text", async () => { + const pdf = buildPdf([ + ["Fenix architecture", "The service is split in two."], + ["Storage", "Postgres, one schema per tenant."], + ]); + const pages = await extractPdfPages(new Uint8Array(pdf)); + + expect(pages.map((p) => p.page)).toEqual([1, 2]); + expect(pages[0]!.text).toContain("Fenix architecture"); + expect(pages[0]!.text).toContain("The service is split in two."); + expect(pages[1]!.text).toContain("Postgres, one schema per tenant."); + // Page two's text must not have leaked into page one, or every citation + // after the first page points at the wrong place. + expect(pages[0]!.text).not.toContain("Postgres"); + }); + + it("keeps a page with no extractable text, so later anchors do not shift", async () => { + const pages = await extractPdfPages(new Uint8Array(buildPdf([["one"], [], ["three"]]))); + expect(pages.map((p) => p.page)).toEqual([1, 2, 3]); + expect(pages[1]!.text).toBe(""); + expect(pages[2]!.text).toContain("three"); + }); + + it("leaves the caller's buffer readable after extraction", async () => { + // pdfjs transfers the buffer it is handed; uploadPdfSource still needs the + // bytes afterwards to preserve the original. + const pdf = buildPdf([["only page"]]); + const bytes = new Uint8Array(pdf); + await extractPdfPages(bytes); + expect(bytes.byteLength).toBeGreaterThan(0); + expect(Buffer.from(bytes).subarray(0, 5).toString("latin1")).toBe("%PDF-"); + }); + }); + + describe("renderPdfText", () => { + it("puts every page under its own anchor, in order", () => { + const md = renderPdfText([ + { page: 1, text: "first" }, + { page: 2, text: "second" }, + ]); + expect(md).toBe("## p1\n\nfirst\n\n## p2\n\nsecond"); + }); + + it("still emits the anchor of a page with no text", () => { + expect( + renderPdfText([ + { page: 1, text: "" }, + { page: 2, text: "b" }, + ]), + ).toBe("## p1\n\n## p2\n\nb"); + }); + }); + + describe("uploadPdfSource", () => { + it("preserves the original and writes text.md with the page anchors", async () => { + const pdf = buildPdf([["Fenix architecture"], ["Storage notes"]]); + const { id, pages } = await uploadPdfSource(root, "Arquitetura Fenix.pdf", pdf); + + expect(id).toBe("arquitetura-fenix.pdf"); + expect(pages).toBe(2); + + const dir = join(root, "raw", id); + expect(existsSync(join(dir, "source.pdf"))).toBe(true); + // The preserved original is the bytes that came in, not a re-encoding. + expect(readFileSync(join(dir, "source.pdf")).equals(pdf)).toBe(true); + + const text = readFileSync(join(dir, "text.md"), "utf8"); + expect(text).toContain("## p1"); + expect(text).toContain("Fenix architecture"); + expect(text).toContain("## p2"); + expect(text).toContain("Storage notes"); + expect(text.endsWith("\n")).toBe(true); + + const manifest = readManifest(root, id); + expect(manifest.kind).toBe("file"); + expect(manifest.title).toBe("Arquitetura Fenix.pdf"); + expect(manifest.original).toBe("Arquitetura Fenix.pdf"); + }); + + it("refuses a filename already taken rather than inventing a suffix", async () => { + const pdf = buildPdf([["x"]]); + await uploadPdfSource(root, "report.pdf", pdf); + await expect(uploadPdfSource(root, "report.pdf", pdf)).rejects.toThrow(/already exists/); + }); + }); +}); diff --git a/packages/access/tests/sources-upload.spec.ts b/packages/access/tests/sources-upload.spec.ts new file mode 100644 index 0000000..a3e5ba0 --- /dev/null +++ b/packages/access/tests/sources-upload.spec.ts @@ -0,0 +1,144 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + MAX_SOURCE_BYTES, + ingestSource, + recogniseSource, + recognisedExtensions, +} from "../src/sources/upload.js"; +import { listSources } from "../src/sources/manifest.js"; +import { buildPdf, buildDocx } from "./fixtures/documents.js"; + +function tempProject(): string { + const root = mkdtempSync(join(tmpdir(), "ow-upload-")); + mkdirSync(join(root, "raw"), { recursive: true }); + return root; +} + +describe("the upload door (3.5, 3.7)", () => { + let root: string; + beforeEach(() => (root = tempProject())); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + describe("recogniseSource", () => { + it("recognises the three source formats by extension", () => { + expect(recogniseSource("notes.md")).toBe("text"); + expect(recogniseSource("notes.markdown")).toBe("text"); + expect(recogniseSource("notes.txt")).toBe("text"); + expect(recogniseSource("architecture.pdf")).toBe("pdf"); + expect(recogniseSource("requisitos.docx")).toBe("docx"); + }); + + it("ignores the case of the extension", () => { + expect(recogniseSource("ARCHITECTURE.PDF")).toBe("pdf"); + expect(recogniseSource("Notes.Md")).toBe("text"); + }); + + it("recognises nothing else, including a file with no extension", () => { + expect(recogniseSource("recording.mp3")).toBeNull(); + expect(recogniseSource("archive.zip")).toBeNull(); + expect(recogniseSource("legacy.doc")).toBeNull(); + expect(recogniseSource("README")).toBeNull(); + }); + + it("does not read a leading dot as an extension", () => { + // `.gitignore` is a name, not an extension. + expect(recogniseSource(".gitignore")).toBeNull(); + }); + + it("lists what it accepts, so a caller can advertise it", () => { + expect(recognisedExtensions()).toContain(".pdf"); + expect(recognisedExtensions()).toContain(".docx"); + expect(recognisedExtensions()).toContain(".md"); + }); + }); + + describe("ingestSource", () => { + it("ingests markdown through the text adapter", async () => { + const outcome = await ingestSource(root, "Notas da reunião.md", Buffer.from("# Notas\n")); + expect(outcome).toMatchObject({ ok: true, format: "text", id: "notas-da-reuniao.md" }); + if (!outcome.ok) throw new Error("expected the upload to land"); + expect(readFileSync(join(root, "raw", outcome.id, "text.md"), "utf8")).toBe("# Notas\n"); + }); + + it("ingests a PDF through the PDF adapter, anchors and all", async () => { + const outcome = await ingestSource(root, "arch.pdf", buildPdf([["one"], ["two"]])); + expect(outcome).toMatchObject({ ok: true, format: "pdf", id: "arch.pdf" }); + if (!outcome.ok) throw new Error("expected the upload to land"); + const text = readFileSync(join(root, "raw", outcome.id, "text.md"), "utf8"); + expect(text).toContain("## p1"); + expect(text).toContain("## p2"); + }); + + it("ingests a DOCX through the DOCX adapter, hierarchy and all", async () => { + const docx = buildDocx([{ text: "Title", style: "Heading1" }, { text: "Body" }]); + const outcome = await ingestSource(root, "spec.docx", docx); + expect(outcome).toMatchObject({ ok: true, format: "docx", id: "spec.docx" }); + if (!outcome.ok) throw new Error("expected the upload to land"); + expect(readFileSync(join(root, "raw", outcome.id, "text.md"), "utf8")).toBe( + "# Title\n\nBody\n", + ); + }); + + it("reports an unrecognised format instead of throwing, and lists what it knows", async () => { + const outcome = await ingestSource(root, "meeting.mp3", Buffer.from("ID3")); + expect(outcome.ok).toBe(false); + if (outcome.ok) throw new Error("expected a refusal"); + expect(outcome.format).toBeNull(); + expect(outcome.reason).toContain(".mp3"); + expect(outcome.reason).toContain(".pdf"); + }); + + it("says what to do about a legacy .doc rather than only refusing it", async () => { + const outcome = await ingestSource(root, "old.doc", Buffer.from("\xd0\xcf")); + expect(outcome.ok).toBe(false); + if (outcome.ok) throw new Error("expected a refusal"); + expect(outcome.reason).toContain("save it as .docx"); + }); + + it("reports a name already taken as this file's reason, not as a throw", async () => { + await ingestSource(root, "notes.md", Buffer.from("first")); + const outcome = await ingestSource(root, "notes.md", Buffer.from("second")); + expect(outcome.ok).toBe(false); + if (outcome.ok) throw new Error("expected a refusal"); + expect(outcome.reason).toMatch(/already exists/); + // The first upload is untouched: a refusal never overwrites a source. + expect(readFileSync(join(root, "raw", "notes.md", "text.md"), "utf8")).toBe("first\n"); + }); + + it("refuses a file over the size ceiling before any reader sees it", async () => { + const huge = Buffer.alloc(MAX_SOURCE_BYTES + 1); + const outcome = await ingestSource(root, "huge.pdf", huge); + expect(outcome.ok).toBe(false); + if (outcome.ok) throw new Error("expected a refusal"); + expect(outcome.reason).toMatch(/over the \d+-byte limit/); + expect(listSources(root)).toEqual([]); + }); + + it("registers under the basename when handed a whole path", async () => { + // The drag-and-drop surface of 3.5 naturally has a full path. Recognition + // looked past the directories and id derivation did not, so this used to + // land as `home-u-documents-report-pdf`. + const outcome = await ingestSource(root, "/home/u/Documents/Report.pdf", buildPdf([["x"]])); + expect(outcome).toMatchObject({ ok: true, id: "report.pdf" }); + }); + + it("reports a document the reader will not open as this file's reason", async () => { + const outcome = await ingestSource(root, "broken.pdf", Buffer.from("not a pdf at all")); + expect(outcome.ok).toBe(false); + if (outcome.ok) throw new Error("expected a refusal"); + expect(outcome.format).toBe("pdf"); + expect(outcome.reason.length).toBeGreaterThan(0); + }); + + it("registers no source when the document could not be read", async () => { + await ingestSource(root, "broken.pdf", Buffer.from("not a pdf at all")); + // A half-registered source — a directory with a manifest and no text — + // would be a permanent, immutable mistake under raw/, so extraction runs + // before registration rather than after it. + expect(listSources(root)).toEqual([]); + }); + }); +}); diff --git a/packages/access/tsconfig.json b/packages/access/tsconfig.json index 71351fd..fd5f59a 100644 --- a/packages/access/tsconfig.json +++ b/packages/access/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "../../tsconfig.base.json", - "include": ["src", "tests"] + "include": ["src", "tests", "types"] } diff --git a/packages/access/types/mammoth.d.ts b/packages/access/types/mammoth.d.ts new file mode 100644 index 0000000..1999ba9 --- /dev/null +++ b/packages/access/types/mammoth.d.ts @@ -0,0 +1,30 @@ +/** + * mammoth ships no type declarations and there is no `@types/mammoth`. This + * declares the slice of its API `sources/docx.ts` uses, and nothing more: a + * hand-written declaration that describes the whole library would be a second + * copy of someone else's contract, drifting silently. + */ +declare module "mammoth" { + export interface ConvertInput { + buffer: Buffer; + } + + export interface ConvertMessage { + type: "warning" | "error"; + message: string; + } + + export interface ConvertResult { + value: string; + messages: ConvertMessage[]; + } + + export function convertToHtml(input: ConvertInput): Promise; + export function extractRawText(input: ConvertInput): Promise; + + const mammoth: { + convertToHtml: typeof convertToHtml; + extractRawText: typeof extractRawText; + }; + export default mammoth; +} diff --git a/plans/open-wiki.md b/plans/open-wiki.md index f353200..878226c 100644 --- a/plans/open-wiki.md +++ b/plans/open-wiki.md @@ -1,6 +1,6 @@ --- autonomy: auto -ci: no-wait +ci: wait --- # Open Wiki — desktop @@ -116,11 +116,13 @@ fenix/ a project — usually a repository the user alre - [x] 3.1 (TDD) Register a source in `raw//` with a `manifest.json` carrying its title, the original preserved, and the directory marked immutable once written — the id is derived from the source's name and frozen there, per `adr:0011-sources-are-named-by-what-they-are` - [x] 3.2 (Unit) Upload Markdown and plain text: copy into `raw/` and normalise to `text.md` -- [ ] 3.3 (Unit) Upload a PDF: extract the text to `text.md`, keeping the page number as a provenance anchor -- [ ] 3.4 (Unit) Upload a DOCX: extract the text and the heading hierarchy to `text.md` +- [x] 3.3 (Unit) Upload a PDF: extract the text to `text.md`, keeping the page number as a provenance anchor +- [x] 3.4 (Unit) Upload a DOCX: extract the text and the heading hierarchy to `text.md` + - Requirement gap a review surfaced, to settle in group 7 with the rest of provenance: a DOCX records no pagination, so this writes the heading hierarchy and **no** page anchor — inventing a `p` would be a number that looks like provenance and points nowhere. But 5.4 already shipped, and its `FILE_FRAGMENT` accepts only `p` for a `src://` citation. So a DOCX source is citable only as `src://#p1`, which resolves to the source but to no place inside its `text.md`. Closing it means either a fragment form for a structural anchor (a heading slug, checked against the headings in `text.md`) or accepting that a DOCX is cited whole. The MVP does not depend on it: the path that matters end to end is markdown and PDF. - [ ] 3.5 (Unit) Drag files onto the window and see what was recognised and what was not - [x] 3.6 (TDD) Derive the id: lowercase, accents folded, anything outside `[a-z0-9]` collapsed to one `-`, and refuse a filename already taken in this project instead of inventing a suffix -- [ ] 3.7 (Unit) Watch `raw/_inbox/` and ingest what lands there through the same path as 3.1 — the way an agent hands over material it fetched, now that no MCP tool ingests. The inbox is the one mutable thing under `raw/`: it is a doorway, emptied by ingestion, and it is not a source, so nothing enumerates it, cites it or reports it uncited +- [x] 3.7 (Unit) Watch `raw/_inbox/` and ingest what lands there through the same path as 3.1 — the way an agent hands over material it fetched, now that no MCP tool ingests. The inbox is the one mutable thing under `raw/`: it is a doorway, emptied by ingestion, and it is not a source, so nothing enumerates it, cites it or reports it uncited + - The watcher is built and tested but **nothing runs it yet** — 8.2's shell is what will hold it open. Until then the doorway works through `drainInbox`, called by hand. A reader should not take the ticked box to mean a file dropped into `raw/_inbox/` is picked up by a running process today. ## 4 — Sources: audio recording diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57daaf0..a80d7f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,15 @@ importers: packages/access: dependencies: + chokidar: + specifier: ^5.0.0 + version: 5.0.0 + mammoth: + specifier: ^1.12.0 + version: 1.12.0 + pdfjs-dist: + specifier: ^6.2.108 + version: 6.2.108 yaml: specifier: ^2.7.0 version: 2.9.0 @@ -346,6 +355,76 @@ packages: '@cfworker/json-schema': optional: true + '@napi-rs/canvas-android-arm64@1.0.3': + resolution: {integrity: sha512-7kSCdUhoXiO+AaIMXdBGdtp6EctZNkmF62Rea/BmVQlwKaM3bBhOzyGUzxyxz9dv5vdBfpyAaxhSRSJF4kqK4A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/canvas-darwin-arm64@1.0.3': + resolution: {integrity: sha512-ds14V1BPagLszQyaDTeggny5fNeTCqsUQ5QhFj9VDxSEfzrVxXtdbR0LoFyKa0Siaaw8KvqSk4t7k/WoZJwvbg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/canvas-darwin-x64@1.0.3': + resolution: {integrity: sha512-qof3LRAAycmkV2I1izZo9RoSHF8kCQr5O05sFwv0jK8rSdYV6KHVwimo6Qb7RxZj40WHKbLHm5JDaUF0o5XUAA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.3': + resolution: {integrity: sha512-FU2kKZLmolHA9+KcUA+l1+xH3WTLUUTQDU/kLv9SEUr2TrRPu94aytOeizFJDHPs/QBcw4QL1mCQhetQXYBbag==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/canvas-linux-arm64-gnu@1.0.3': + resolution: {integrity: sha512-GVSjntxKeA+/y/ZKf1F+cmUw1WeIkE5aMRPqnZUlBTBvBcrvgWccJAWuYCKPX4QJQwZILIIwhgdAbl51yj6fpA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-arm64-musl@1.0.3': + resolution: {integrity: sha512-J51oK/axyZ13kxycumSMfLiDZMdWdOVvqDFI28BpuViZHE3A0bQfr8B5vg8YnPEnqLD3BSn1hkdlh2buspEcNQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-riscv64-gnu@1.0.3': + resolution: {integrity: sha512-CtQgQjoVTX67jS9XuCTtJ40Sl7wRLMguoFnnGnfDmCWf7kzKFZVwj5ynqUOIGKFMSB61ZCuQlwPvVNxYTTseaw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/canvas-linux-x64-gnu@1.0.3': + resolution: {integrity: sha512-jtfzAHFp+FRaR7zGT4jyCe6wUgAG/dVb5A4Apd8FY9jKarntDfUAlJXscugiH7ZF5kKnu7/lHFk9LaDPcrGEVQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-linux-x64-musl@1.0.3': + resolution: {integrity: sha512-xTzaUCKUHTY4bCGadeeRZggbRVbGUT1petg7Z8r9AJR2+D9Bqu6nQAgqBGC6D47tA70LjaaaLTrJ7wNY1T74dg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-win32-arm64-msvc@1.0.3': + resolution: {integrity: sha512-ktVLuBkI6QVOm5BwO/WbdGwxgeetAMJa7TTmR8qBarXF0OU2NKjvjUtPJAl2y8t+zBRczJl/1VOl9gua6WcK2g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/canvas-win32-x64-msvc@1.0.3': + resolution: {integrity: sha512-SGhlQ8bDjL1Cz2KnsKMasr/5sTcwG/SZkB6WCJxLsmSm/3aS2C+3p39bA7iZ2/94+NkVDySZfbiGoaSZSFHYxA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/canvas@1.0.3': + resolution: {integrity: sha512-OlI657a5XXvKGFX7kNeIzJ8rO7IXt87Mqu2H8rXE46viAuOfum/JA7ysX7+eBhxNKznT+RCZh418mndlcFX3+w==} + engines: {node: '>= 10'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -587,6 +666,10 @@ packages: '@vitest/utils@3.2.7': resolution: {integrity: sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==} + '@xmldom/xmldom@0.8.13': + resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} + engines: {node: '>=10.0.0'} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -631,6 +714,9 @@ packages: resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} engines: {node: '>=12'} + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -648,6 +734,12 @@ packages: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + bluebird@3.4.7: + resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==} + body-parser@2.3.0: resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} engines: {node: '>=18'} @@ -694,6 +786,10 @@ packages: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} engines: {node: '>= 16'} + chokidar@5.0.0: + resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==} + engines: {node: '>= 20.19.0'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -724,6 +820,9 @@ packages: resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} engines: {node: '>= 0.6'} + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + cors@2.8.6: resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} engines: {node: '>= 0.10'} @@ -752,6 +851,12 @@ packages: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} + dingbat-to-unicode@1.0.1: + resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==} + + duck@0.1.12: + resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==} + dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -1004,6 +1109,9 @@ packages: resolution: {integrity: sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==} engines: {node: '>= 4'} + immediate@3.0.6: + resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} + import-fresh@3.3.1: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} @@ -1038,6 +1146,9 @@ packages: is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -1088,6 +1199,9 @@ packages: json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + jszip@3.10.1: + resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -1095,6 +1209,9 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} + lie@3.3.0: + resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==} + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -1102,6 +1219,9 @@ packages: lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + lop@0.4.2: + resolution: {integrity: sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==} + loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} @@ -1118,6 +1238,11 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} + mammoth@1.12.0: + resolution: {integrity: sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==} + engines: {node: '>=12.0.0'} + hasBin: true + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -1183,6 +1308,9 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + option@0.2.4: + resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -1198,6 +1326,9 @@ packages: package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + pako@1.0.11: + resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -1210,6 +1341,10 @@ packages: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} + path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -1228,6 +1363,10 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pdfjs-dist@6.2.108: + resolution: {integrity: sha512-YxFb+SQcodN2rnX9Tn3dHYlqfb7NjlzzfONPpJd+AKoKtUjEdevTfbC07d5TcczzOK6261auRkP/M8OBHs9vFQ==} + engines: {node: '>=22.13.0 || >=24'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -1252,6 +1391,9 @@ packages: engines: {node: '>=14'} hasBin: true + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -1272,6 +1414,13 @@ packages: resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} engines: {node: '>= 0.10'} + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + + readdirp@5.0.0: + resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==} + engines: {node: '>= 20.19.0'} + require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} @@ -1289,6 +1438,9 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -1305,6 +1457,9 @@ packages: resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} engines: {node: '>= 18'} + setimmediate@1.0.5: + resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -1343,6 +1498,9 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -1361,6 +1519,9 @@ packages: resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} engines: {node: '>=12'} + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -1436,6 +1597,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + underscore@1.13.8: + resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -1446,6 +1610,9 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} @@ -1548,6 +1715,10 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + xmlbuilder@10.1.1: + resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==} + engines: {node: '>=4.0'} + yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} @@ -1778,6 +1949,54 @@ snapshots: transitivePeerDependencies: - supports-color + '@napi-rs/canvas-android-arm64@1.0.3': + optional: true + + '@napi-rs/canvas-darwin-arm64@1.0.3': + optional: true + + '@napi-rs/canvas-darwin-x64@1.0.3': + optional: true + + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.3': + optional: true + + '@napi-rs/canvas-linux-arm64-gnu@1.0.3': + optional: true + + '@napi-rs/canvas-linux-arm64-musl@1.0.3': + optional: true + + '@napi-rs/canvas-linux-riscv64-gnu@1.0.3': + optional: true + + '@napi-rs/canvas-linux-x64-gnu@1.0.3': + optional: true + + '@napi-rs/canvas-linux-x64-musl@1.0.3': + optional: true + + '@napi-rs/canvas-win32-arm64-msvc@1.0.3': + optional: true + + '@napi-rs/canvas-win32-x64-msvc@1.0.3': + optional: true + + '@napi-rs/canvas@1.0.3': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 1.0.3 + '@napi-rs/canvas-darwin-arm64': 1.0.3 + '@napi-rs/canvas-darwin-x64': 1.0.3 + '@napi-rs/canvas-linux-arm-gnueabihf': 1.0.3 + '@napi-rs/canvas-linux-arm64-gnu': 1.0.3 + '@napi-rs/canvas-linux-arm64-musl': 1.0.3 + '@napi-rs/canvas-linux-riscv64-gnu': 1.0.3 + '@napi-rs/canvas-linux-x64-gnu': 1.0.3 + '@napi-rs/canvas-linux-x64-musl': 1.0.3 + '@napi-rs/canvas-win32-arm64-msvc': 1.0.3 + '@napi-rs/canvas-win32-x64-msvc': 1.0.3 + optional: true + '@pkgjs/parseargs@0.11.0': optional: true @@ -2023,6 +2242,8 @@ snapshots: loupe: 3.2.1 tinyrainbow: 2.0.0 + '@xmldom/xmldom@0.8.13': {} + accepts@2.0.0: dependencies: mime-types: 3.0.2 @@ -2062,6 +2283,10 @@ snapshots: ansi-styles@6.2.3: {} + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + argparse@2.0.1: {} assertion-error@2.0.1: {} @@ -2076,6 +2301,10 @@ snapshots: balanced-match@4.0.4: {} + base64-js@1.5.1: {} + + bluebird@3.4.7: {} + body-parser@2.3.0: dependencies: bytes: 3.1.2 @@ -2134,6 +2363,10 @@ snapshots: check-error@2.1.3: {} + chokidar@5.0.0: + dependencies: + readdirp: 5.0.0 + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -2152,6 +2385,8 @@ snapshots: cookie@0.7.2: {} + core-util-is@1.0.3: {} + cors@2.8.6: dependencies: object-assign: 4.1.1 @@ -2173,6 +2408,12 @@ snapshots: depd@2.0.0: {} + dingbat-to-unicode@1.0.1: {} + + duck@0.1.12: + dependencies: + underscore: 1.13.8 + dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -2477,6 +2718,8 @@ snapshots: ignore@7.0.6: {} + immediate@3.0.6: {} + import-fresh@3.3.1: dependencies: parent-module: 1.0.1 @@ -2500,6 +2743,8 @@ snapshots: is-promise@4.0.0: {} + isarray@1.0.0: {} + isexe@2.0.0: {} istanbul-lib-coverage@3.2.2: {} @@ -2549,6 +2794,13 @@ snapshots: json-stable-stringify-without-jsonify@1.0.1: {} + jszip@3.10.1: + dependencies: + lie: 3.3.0 + pako: 1.0.11 + readable-stream: 2.3.8 + setimmediate: 1.0.5 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -2558,12 +2810,22 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 + lie@3.3.0: + dependencies: + immediate: 3.0.6 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 lodash.merge@4.6.2: {} + lop@0.4.2: + dependencies: + duck: 0.1.12 + option: 0.2.4 + underscore: 1.13.8 + loupe@3.2.1: {} lru-cache@10.4.3: {} @@ -2582,6 +2844,19 @@ snapshots: dependencies: semver: 7.8.5 + mammoth@1.12.0: + dependencies: + '@xmldom/xmldom': 0.8.13 + argparse: 1.0.10 + base64-js: 1.5.1 + bluebird: 3.4.7 + dingbat-to-unicode: 1.0.1 + jszip: 3.10.1 + lop: 0.4.2 + path-is-absolute: 1.0.1 + underscore: 1.13.8 + xmlbuilder: 10.1.1 + math-intrinsics@1.1.0: {} media-typer@1.1.1: {} @@ -2628,6 +2903,8 @@ snapshots: dependencies: wrappy: 1.0.2 + option@0.2.4: {} + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -2647,6 +2924,8 @@ snapshots: package-json-from-dist@1.0.1: {} + pako@1.0.11: {} + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -2655,6 +2934,8 @@ snapshots: path-exists@4.0.0: {} + path-is-absolute@1.0.1: {} + path-key@3.1.1: {} path-scurry@1.11.1: @@ -2668,6 +2949,10 @@ snapshots: pathval@2.0.1: {} + pdfjs-dist@6.2.108: + optionalDependencies: + '@napi-rs/canvas': 1.0.3 + picocolors@1.1.1: {} picomatch@4.0.5: {} @@ -2684,6 +2969,8 @@ snapshots: prettier@3.9.6: {} + process-nextick-args@2.0.1: {} + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -2705,6 +2992,18 @@ snapshots: iconv-lite: 0.7.3 unpipe: 1.0.0 + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + + readdirp@5.0.0: {} + require-from-string@2.0.2: {} resolve-from@4.0.0: {} @@ -2750,6 +3049,8 @@ snapshots: transitivePeerDependencies: - supports-color + safe-buffer@5.1.2: {} + safer-buffer@2.1.2: {} semver@7.8.5: {} @@ -2779,6 +3080,8 @@ snapshots: transitivePeerDependencies: - supports-color + setimmediate@1.0.5: {} + setprototypeof@1.2.0: {} shebang-command@2.0.0: @@ -2821,6 +3124,8 @@ snapshots: source-map-js@1.2.1: {} + sprintf-js@1.0.3: {} + stackback@0.0.2: {} statuses@2.0.2: {} @@ -2839,6 +3144,10 @@ snapshots: emoji-regex: 9.2.2 strip-ansi: 7.2.0 + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -2907,6 +3216,8 @@ snapshots: typescript@5.9.3: {} + underscore@1.13.8: {} + undici-types@6.21.0: {} unpipe@1.0.0: {} @@ -2915,6 +3226,8 @@ snapshots: dependencies: punycode: 2.3.1 + util-deprecate@1.0.2: {} + vary@1.1.2: {} vite-node@3.2.4(@types/node@22.20.1)(yaml@2.9.0): @@ -3017,6 +3330,8 @@ snapshots: wrappy@1.0.2: {} + xmlbuilder@10.1.1: {} + yaml@2.9.0: {} yocto-queue@0.1.0: {} From 39ef20913ad2604f749d56e0c9afbb88ba5ccbca Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sat, 1 Aug 2026 08:23:33 +0000 Subject: [PATCH 2/3] test(access): let the inbox suite run on a Windows account without the symlink privilege The repo already established this in paths.spec.ts: creating a symlink is a privilege a Windows account may not have, and that failure is a different thing from the containment behaviour under test. Windows is the only platform this product supports, so the suite has to run there for a developer who is not elevated. Deleting a directory chokidar is watching is the same kind of platform difference, and is guarded the same way. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016iMM93Wk43o44V5J2AxPgL --- packages/access/tests/sources-inbox.spec.ts | 30 ++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/access/tests/sources-inbox.spec.ts b/packages/access/tests/sources-inbox.spec.ts index 136560b..0f17e2e 100644 --- a/packages/access/tests/sources-inbox.spec.ts +++ b/packages/access/tests/sources-inbox.spec.ts @@ -42,6 +42,22 @@ async function until(predicate: () => boolean, timeoutMs = 5000): Promise } } +/** + * Create a symlink, or return false when this account cannot. A Windows + * account without the symlink privilege is a different failure from the + * behaviour under test, and the repo already treats it that way in + * `paths.spec.ts` — Windows is the platform this product supports, so the + * suite has to run there for a developer who is not elevated. + */ +function trySymlink(target: string, path: string): boolean { + try { + symlinkSync(target, path); + return true; + } catch { + return false; + } +} + describe("the raw/_inbox doorway (3.7)", () => { let root: string; beforeEach(() => (root = tempProject())); @@ -111,7 +127,7 @@ describe("the raw/_inbox doorway (3.7)", () => { it("refuses to follow a symbolic link out of the project", async () => { const outside = join(root, "secret.md"); writeFileSync(outside, "# not the agent's to publish\n"); - symlinkSync(outside, join(root, "raw", INBOX, "innocuous.md")); + if (!trySymlink(outside, join(root, "raw", INBOX, "innocuous.md"))) return; const outcomes = await drainInbox(root); expect(outcomes[0]!.ok).toBe(false); @@ -122,7 +138,7 @@ describe("the raw/_inbox doorway (3.7)", () => { it("keeps draining the rest of the batch after a file it refused", async () => { // One bad entry ending the whole drain would mean a single dropped // symlink silently stops every later file from ever being ingested. - symlinkSync(join(root, "raw"), join(root, "raw", INBOX, "a-link.md")); + if (!trySymlink(join(root, "raw"), join(root, "raw", INBOX, "a-link.md"))) return; writeFileSync(join(root, "raw", INBOX, "z-real.md"), "# Real\n"); const outcomes = await drainInbox(root); @@ -331,10 +347,16 @@ describe("the raw/_inbox doorway (3.7)", () => { try { // Replace raw/ with a link out of the project: the inbox no longer // resolves inside it, and an explicit drain has to say so. - rmSync(join(root, "raw"), { recursive: true, force: true }); + try { + rmSync(join(root, "raw"), { recursive: true, force: true }); + } catch { + // Windows refuses to delete a directory that is being watched. That + // is the platform, not the behaviour under test. + return; + } const elsewhere = mkdtempSync(join(tmpdir(), "ow-elsewhere-")); try { - symlinkSync(elsewhere, join(root, "raw")); + if (!trySymlink(elsewhere, join(root, "raw"))) return; await expect(watcher.drain()).rejects.toThrow(/outside the project/); } finally { rmSync(elsewhere, { recursive: true, force: true }); From 9928812beb827797e3fa131e8a83d5e0d53ec1ef Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sat, 1 Aug 2026 08:55:18 +0000 Subject: [PATCH 3/3] fix(access): close the CodeRabbit findings on the ingestion paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `watchInbox` no longer hangs when chokidar never emits `ready`. The event is not guaranteed after a failure in the initial scan, and a bare await on it means the desktop application never finishes opening the project. It now settles on ready or error, whichever comes first, with a bounded fallback. - A ZIP64 DOCX is reported as ZIP64. It was refused correctly and then described to the user as declaring "Infinity bytes of content". - A numeric character reference above the Unicode range no longer aborts the conversion. `String.fromCodePoint` throws there, and `htmlToMarkdown` is exported, so the HTML need not have come from mammoth. Out-of-range and surrogate-half references stay literal. - `engines.node` is `>=22.13.0`: pdfjs-dist 6.2.108 requires it, and `zlib.crc32` — which the DOCX fixture builds on — landed in 22.2.0. - The inbox tests resolve their temp root. `inboxPath` returns a real path, and `os.tmpdir()` is itself a symlink on macOS, so the assertion compared two spellings of the same directory. - The two platform skips catch only the codes that mean "this platform cannot do that". Catching everything turned an unexpected setup failure into a passing test. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016iMM93Wk43o44V5J2AxPgL --- package.json | 2 +- packages/access/src/sources/docx.ts | 36 +++++++++++++++++---- packages/access/src/sources/inbox.ts | 31 ++++++++++++++++-- packages/access/tests/fixtures/documents.ts | 3 ++ packages/access/tests/sources-docx.spec.ts | 30 ++++++++++++++++- packages/access/tests/sources-inbox.spec.ts | 23 +++++++++---- 6 files changed, 108 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index c332622..faa8574 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "module", "packageManager": "pnpm@10.15.0", "engines": { - "node": ">=22" + "node": ">=22.13.0" }, "scripts": { "test": "pnpm -r --if-present run test", diff --git a/packages/access/src/sources/docx.ts b/packages/access/src/sources/docx.ts index 7eba3a4..c243c71 100644 --- a/packages/access/src/sources/docx.ts +++ b/packages/access/src/sources/docx.ts @@ -23,13 +23,26 @@ import { writeSourceText } from "./ingest.js"; * fires this package on every page write and must not pay for a DOCX reader. */ +/** + * A numeric character reference, or the reference itself when it names no + * character. `String.fromCodePoint` throws above `0x10FFFF`, and this module is + * exported: a caller can hand it HTML mammoth never produced, and one bad + * entity must not abort the whole conversion. + */ +function codePoint(value: number, literal: string): string { + if (!Number.isInteger(value) || value < 0 || value > 0x10ffff) return literal; + // Surrogate halves are not characters on their own. + if (value >= 0xd800 && value <= 0xdfff) return literal; + return String.fromCodePoint(value); +} + /** Decode the entities mammoth emits in text; it escapes nothing else. */ function decodeEntities(text: string): string { return ( text .replace(/ /g, " ") - .replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code))) - .replace(/&#x([0-9a-f]+);/gi, (_, code: string) => String.fromCodePoint(parseInt(code, 16))) + .replace(/&#(\d+);/g, (whole, code: string) => codePoint(Number(code), whole)) + .replace(/&#x([0-9a-f]+);/gi, (whole, code: string) => codePoint(parseInt(code, 16), whole)) .replace(/"/g, '"') .replace(/'/g, "'") .replace(/</g, "<") @@ -299,7 +312,10 @@ const ZIP64_SENTINEL = 0xffffffff; * inflating anything. Returns `null` when the structure cannot be read — a * malformed archive is mammoth's to report, not this guard's to guess at. */ -export function declaredUncompressedSize(zip: Buffer): number | null { +/** A ZIP64 archive: the real sizes live in an extra field this does not parse. */ +export const ZIP64 = Symbol("zip64"); + +export function declaredUncompressedSize(zip: Buffer): number | typeof ZIP64 | null { const start = Math.max(0, zip.length - (22 + 0xffff)); let eocd = -1; for (let i = zip.length - 22; i >= start; i--) { @@ -312,7 +328,7 @@ export function declaredUncompressedSize(zip: Buffer): number | null { const entries = zip.readUInt16LE(eocd + 10); let offset = zip.readUInt32LE(eocd + 16); - if (offset === ZIP64_SENTINEL) return Number.POSITIVE_INFINITY; // ZIP64: refuse + if (offset === ZIP64_SENTINEL) return ZIP64; let total = 0; for (let n = 0; n < entries; n++) { @@ -320,9 +336,9 @@ export function declaredUncompressedSize(zip: Buffer): number | null { if (zip.readUInt32LE(offset) !== CENTRAL_SIGNATURE) return null; const size = zip.readUInt32LE(offset + 24); // A ZIP64 entry hides its real size in an extra field. Rather than parse - // that, treat it as over any ceiling: a document that needs ZIP64 is not a - // document this reads. - if (size === ZIP64_SENTINEL) return Number.POSITIVE_INFINITY; + // that, say so: a document that needs ZIP64 is not one this reads, and + // "declares Infinity bytes" is not a sentence to show anybody. + if (size === ZIP64_SENTINEL) return ZIP64; total += size; offset += 46 + @@ -340,6 +356,12 @@ export function declaredUncompressedSize(zip: Buffer): number | null { */ export async function extractDocxMarkdown(content: Buffer): Promise { const declared = declaredUncompressedSize(content); + if (declared === ZIP64) { + throw new Error( + "this DOCX is a ZIP64 archive, whose entry sizes cannot be checked before they are read — " + + "a document that large is not one this reads", + ); + } if (declared !== null && declared > MAX_DOCX_UNCOMPRESSED_BYTES) { throw new Error( `this DOCX declares ${declared} bytes of content, over the ${MAX_DOCX_UNCOMPRESSED_BYTES}-byte limit — ` + diff --git a/packages/access/src/sources/inbox.ts b/packages/access/src/sources/inbox.ts index 1ef5ce1..d7bcd25 100644 --- a/packages/access/src/sources/inbox.ts +++ b/packages/access/src/sources/inbox.ts @@ -189,6 +189,13 @@ export interface WatchInboxHandlers { onError?: (error: Error) => void; } +/** + * How long to wait for chokidar's initial scan before carrying on without it. + * Generous, because exceeding it means events may be missed; bounded, because + * the alternative is an application that never opens. + */ +const READY_TIMEOUT_MS = 10_000; + export interface WatchInboxOptions { /** * How long a file's size must hold steady before it is considered fully @@ -321,9 +328,29 @@ export async function watchInbox( // Return only once the initial scan is done. Before `ready` chokidar applies // no write-stability check of its own, and a caller that starts watching and // immediately drops a file would otherwise race the scan. + // + // But `ready` is **not guaranteed to arrive**: chokidar can fail during the + // initial scan and never emit it. A bare await hangs the caller for good — + // the desktop application never finishes opening the project. So settle on + // whichever of ready/error comes first, and give up after a bounded wait + // rather than trading one silent failure for a worse one. await new Promise((resolve) => { - if (watcher.closed) resolve(); - else watcher.once("ready", () => resolve()); + if (watcher.closed) { + resolve(); + return; + } + let settled = false; + const done = (): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(done, READY_TIMEOUT_MS); + // Never hold the process open on account of this timer. + timer.unref?.(); + watcher.once("ready", done); + watcher.once("error", done); }); return { diff --git a/packages/access/tests/fixtures/documents.ts b/packages/access/tests/fixtures/documents.ts index dddeafc..d1b27b1 100644 --- a/packages/access/tests/fixtures/documents.ts +++ b/packages/access/tests/fixtures/documents.ts @@ -123,6 +123,9 @@ function escapeXml(text: string): string { .replace(/"/g, """); } +/** The size field a ZIP64 archive writes when the real one lives elsewhere. */ +export const ZIP64_SENTINEL = 0xffffffff; + /** A zip entry: its name, its bytes, and optionally a size it lies about. */ type ZipEntry = readonly [name: string, text: string, declaredSize?: number]; diff --git a/packages/access/tests/sources-docx.spec.ts b/packages/access/tests/sources-docx.spec.ts index 4a70a3e..3853da2 100644 --- a/packages/access/tests/sources-docx.spec.ts +++ b/packages/access/tests/sources-docx.spec.ts @@ -4,13 +4,14 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { MAX_DOCX_UNCOMPRESSED_BYTES, + ZIP64, declaredUncompressedSize, extractDocxMarkdown, htmlToMarkdown, uploadDocxSource, } from "../src/sources/docx.js"; import { readManifest } from "../src/sources/manifest.js"; -import { buildBombDocx, buildDocx } from "./fixtures/documents.js"; +import { ZIP64_SENTINEL, buildBombDocx, buildDocx } from "./fixtures/documents.js"; function tempProject(): string { const root = mkdtempSync(join(tmpdir(), "ow-docx-")); @@ -136,6 +137,15 @@ describe("DOCX upload (3.4)", () => { ); }); + it("leaves a numeric entity that names no character as it found it", () => { + // String.fromCodePoint throws above 0x10FFFF, and this is exported: one + // bad entity from a caller must not abort the whole conversion. + expect(htmlToMarkdown("

a � b

")).toBe("a � b"); + expect(htmlToMarkdown("

a � b

")).toBe("a � b"); + // A valid one still decodes. + expect(htmlToMarkdown("

été

")).toBe("été"); + }); + it("returns nothing for an empty document", () => { expect(htmlToMarkdown("")).toBe(""); }); @@ -200,6 +210,24 @@ describe("DOCX upload (3.4)", () => { it("returns null for something that is not a zip, leaving the reader to say so", () => { expect(declaredUncompressedSize(Buffer.from("not a zip at all"))).toBeNull(); }); + + it("reports a ZIP64 entry as ZIP64 rather than as a number", () => { + // The real size lives in an extra field this does not parse. Returning + // Infinity refused it correctly and then told the user their document + // "declares Infinity bytes of content". + expect(declaredUncompressedSize(buildBombDocx(ZIP64_SENTINEL))).toBe(ZIP64); + }); + }); + + describe("the ZIP64 refusal", () => { + it("says what is wrong without quoting a byte count it does not have", async () => { + const message = await extractDocxMarkdown(buildBombDocx(ZIP64_SENTINEL)).then( + () => "did not throw", + (err: Error) => err.message, + ); + expect(message).toContain("ZIP64"); + expect(message).not.toContain("Infinity"); + }); }); describe("uploadDocxSource", () => { diff --git a/packages/access/tests/sources-inbox.spec.ts b/packages/access/tests/sources-inbox.spec.ts index 0f17e2e..2b1b193 100644 --- a/packages/access/tests/sources-inbox.spec.ts +++ b/packages/access/tests/sources-inbox.spec.ts @@ -6,6 +6,7 @@ import { mkdtempSync, openSync, readFileSync, + realpathSync, rmSync, symlinkSync, truncateSync, @@ -28,7 +29,10 @@ import { scaffold } from "../src/scaffold.js"; import { buildPdf } from "./fixtures/documents.js"; function tempProject(): string { - const root = mkdtempSync(join(tmpdir(), "ow-inbox-")); + // Resolved once, here. `inboxPath` returns the real path (`assertWithin` + // does), and `os.tmpdir()` is itself a symlink on macOS (/var → /private/var), + // so an unresolved root would not compare equal to what the code returns. + const root = realpathSync(mkdtempSync(join(tmpdir(), "ow-inbox-"))); mkdirSync(join(root, "raw", INBOX), { recursive: true }); return root; } @@ -53,8 +57,12 @@ function trySymlink(target: string, path: string): boolean { try { symlinkSync(target, path); return true; - } catch { - return false; + } catch (err) { + // Only the missing-privilege codes are a platform skip. Anything else is a + // real failure and has to fail the test rather than pass it quietly. + const code = (err as NodeJS.ErrnoException).code; + if (code === "EPERM" || code === "EACCES" || code === "ENOSYS") return false; + throw err; } } @@ -349,10 +357,13 @@ describe("the raw/_inbox doorway (3.7)", () => { // resolves inside it, and an explicit drain has to say so. try { rmSync(join(root, "raw"), { recursive: true, force: true }); - } catch { + } catch (err) { // Windows refuses to delete a directory that is being watched. That - // is the platform, not the behaviour under test. - return; + // is the platform, not the behaviour under test — but only for the + // codes that actually mean it. + const code = (err as NodeJS.ErrnoException).code; + if (code === "EBUSY" || code === "EPERM" || code === "ENOTEMPTY") return; + throw err; } const elsewhere = mkdtempSync(join(tmpdir(), "ow-elsewhere-")); try {