diff --git a/README.md b/README.md index ab7f61b46..986735a2c 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ npx skills add iii-hq/iii --all | [`code-runner`](code-runner/) | Rust | Run untrusted Node.js and Python in-process — V8 isolates and CPython-on-WebAssembly behind one run/register_function/teardown API, with no microVM and no /dev/kvm. Code gets a global `iii` and a private scratch directory. | | [`openwiki`](openwiki/) | Node | Source-grounded markdown wiki for any git repository — a lead agent plans the index and writer sub-agents store cited pages via `openwiki::write-page`, with router and heuristic fallback tiers, incremental refresh from git diffs on a per-wiki cron schedule, and a browser UI + JSON API under `/openwiki`. | | [`pdf`](pdf/) | Rust | Read PDFs locally — `pdf::classify` routes text-based versus scanned in tens of milliseconds and names the pages that still need OCR, `pdf::to-markdown` converts with headings, lists and tables intact, and `pdf::extract-items` / `::extract-regions` expose positions and the text inside a box. Ships a console page. | +| [`document`](document/) | Rust | Read office documents locally — `document::to-markdown` converts Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV and text-based PDFs with their structure intact, `document::detect` names a format from its bytes in microseconds, `document::extract-assets` returns the images markdown cannot carry, and `document::ocr` transcribes a scan by rendering its pages through `browser` and reading them with a vision model. | ## SDK diff --git a/browser/README.md b/browser/README.md index a9b93a16b..fe8cfe078 100644 --- a/browser/README.md +++ b/browser/README.md @@ -158,11 +158,18 @@ browser: max_timeout_ms: 120000 # ceiling; caller timeout_ms clamped DOWN to this idle_stop_ms: 300000 # stop sessions idle this long; 0 disables screenshot_quality: 60 # JPEG quality 1-100 - allowed_schemes: [http, https] + allowed_schemes: [http, https, file] # `file` lets a local document be rendered; see below max_snapshot_nodes: 2000 # a11y outline size cap allow_attach: false # true = allow sessions::attach into a running browser's real profile ``` +`file` is on the default scheme list so a local document can be opened and +rendered, which is how `document::ocr` gets pixels out of a scanned PDF. It is +worth knowing what that permits: navigation is not checked against a session's +filesystem scope the way the workers that read files directly are, so anything +that can reach `browser::navigate` can open any file this process can read. +Narrow the list on a shared machine. + ## Custom trigger types Sibling workers (and the console UI) can subscribe to session activity. All diff --git a/browser/src/config.rs b/browser/src/config.rs index cacd0e555..83e63316d 100644 --- a/browser/src/config.rs +++ b/browser/src/config.rs @@ -47,6 +47,14 @@ pub struct WorkerConfig { /// JPEG quality for `browser::screenshot` (1-100). pub screenshot_quality: u64, /// URL schemes `browser::navigate` accepts. + /// + /// `file` ships enabled so a local document can be opened and rendered — + /// the path `document::ocr` takes to read a scanned PDF, which has nowhere + /// else to get pixels from. Note what that permits: unlike the workers that + /// read files directly, navigation is not checked against a session's + /// filesystem scope, so any caller that reaches `browser::navigate` can + /// open any file this process can read. Narrow the list on a shared or + /// multi-tenant machine. pub allowed_schemes: Vec, /// Maximum nodes serialized by `browser::snapshot` before truncation. pub max_snapshot_nodes: u64, @@ -71,7 +79,7 @@ impl Default for WorkerConfig { max_timeout_ms: 120_000, idle_stop_ms: 300_000, screenshot_quality: 60, - allowed_schemes: vec!["http".to_string(), "https".to_string()], + allowed_schemes: vec!["http".to_string(), "https".to_string(), "file".to_string()], max_snapshot_nodes: 2_000, allow_attach: false, } @@ -128,7 +136,7 @@ mod tests { assert_eq!(c.max_timeout_ms, 120_000); assert_eq!(c.idle_stop_ms, 300_000); assert_eq!(c.screenshot_quality, 60); - assert_eq!(c.allowed_schemes, vec!["http", "https"]); + assert_eq!(c.allowed_schemes, vec!["http", "https", "file"]); assert_eq!(c.max_snapshot_nodes, 2_000); assert!(!c.allow_attach); } diff --git a/browser/src/functions/sessions.rs b/browser/src/functions/sessions.rs index 10146b997..bb7b11c99 100644 --- a/browser/src/functions/sessions.rs +++ b/browser/src/functions/sessions.rs @@ -87,8 +87,22 @@ mod tests { let cfg = WorkerConfig::default(); assert!(check_scheme(&cfg, "http://localhost:3000").is_ok()); assert!(check_scheme(&cfg, "https://example.com/a?b=c").is_ok()); - assert!(check_scheme(&cfg, "file:///etc/passwd").is_err()); + // `file` ships enabled so a local document can be rendered. + assert!(check_scheme(&cfg, "file:///tmp/report.pdf").is_ok()); assert!(check_scheme(&cfg, "chrome://settings").is_err()); assert!(check_scheme(&cfg, "not a url").is_err()); } + + /// The list is what gates navigation, so an operator narrowing it has to + /// actually close the door — including on the scheme that now ships open. + #[test] + fn a_narrowed_list_still_refuses_what_it_drops() { + let cfg = WorkerConfig { + allowed_schemes: vec!["https".to_string()], + ..WorkerConfig::default() + }; + assert!(check_scheme(&cfg, "https://example.com").is_ok()); + assert!(check_scheme(&cfg, "file:///etc/passwd").is_err()); + assert!(check_scheme(&cfg, "http://example.com").is_err()); + } } diff --git a/console/web/src/components/chat/AttachmentButton.tsx b/console/web/src/components/chat/AttachmentButton.tsx index ee3d3b618..f85681495 100644 --- a/console/web/src/components/chat/AttachmentButton.tsx +++ b/console/web/src/components/chat/AttachmentButton.tsx @@ -1,6 +1,6 @@ import { Paperclip } from 'lucide-react' import { useRef } from 'react' -import { uid } from '@/hooks/use-conversations' +import { attachmentsFromFiles } from '@/lib/attachments/from-files' import { cn } from '@/lib/utils' import type { Attachment } from '@/types/chat' @@ -10,20 +10,6 @@ interface AttachmentButtonProps { className?: string } -const MAX_PREVIEW_BYTES = 1_000_000 - -function readPreview(file: File): Promise { - if (file.size > MAX_PREVIEW_BYTES) return Promise.resolve(undefined) - if (!/^(image|text)\//.test(file.type)) return Promise.resolve(undefined) - return new Promise((resolve) => { - const reader = new FileReader() - reader.onload = () => - resolve(typeof reader.result === 'string' ? reader.result : undefined) - reader.onerror = () => resolve(undefined) - reader.readAsDataURL(file) - }) -} - export function AttachmentButton({ onAttach, disabled, @@ -34,19 +20,7 @@ export function AttachmentButton({ const handlePick = async (e: React.ChangeEvent) => { const files = Array.from(e.target.files ?? []) if (files.length === 0) return - const attachments: Attachment[] = await Promise.all( - files.map(async (f) => ({ - id: uid(), - name: f.name, - size: f.size, - type: f.type || 'application/octet-stream', - dataUrl: await readPreview(f), - // Kept so the send path can hand the bytes to a worker that reads this - // kind of file (PDFs go through `pdf::to-markdown`). Not persisted. - file: f, - })), - ) - onAttach(attachments) + onAttach(await attachmentsFromFiles(files)) /* allow re-picking the same file */ e.target.value = '' } diff --git a/console/web/src/components/chat/ChatView.tsx b/console/web/src/components/chat/ChatView.tsx index b1ef63d4f..03edf5f17 100644 --- a/console/web/src/components/chat/ChatView.tsx +++ b/console/web/src/components/chat/ChatView.tsx @@ -23,9 +23,13 @@ import { import { useLiveAnnouncer } from '@/hooks/use-live-announcer' import { useWorktreeBinding } from '@/hooks/use-worktree-binding' import { useWorktreeEvents } from '@/hooks/use-worktree-events' +import { expandAttachments, hasExpandableAttachments } from '@/lib/attachments' import type { ChatBackend } from '@/lib/backend' import { approvalBelongsToConversationTree } from '@/lib/backend/approval-events-live' -import { predictedUserEntryId } from '@/lib/backend/harness-send' +import { + type HarnessImageBlock, + predictedUserEntryId, +} from '@/lib/backend/harness-send' import { serialRefresh } from '@/lib/backend/serial-refresh' import { mergeFiredTriggers, @@ -41,11 +45,6 @@ import { useConversationsCtxOptional } from '@/lib/conversations-context' import { syncEditorWorkspace } from '@/lib/editor-sync' import { expandFileMentions, parseFileMentions } from '@/lib/file-mentions' import { formatStopReason } from '@/lib/format-stop-reason' -import { - expandPdfAttachments, - isPdfAttachment, - summaryLabel, -} from '@/lib/pdf-attachments' import { newMessageId } from '@/lib/session-id' import { useExtSessionChips, useExtSessionTurnSummaries } from '@/lib/ui-slots' import { cn } from '@/lib/utils' @@ -210,6 +209,15 @@ export function ChatView({ const harnessBlockedRef = useRef(harnessBlocked) harnessBlockedRef.current = harnessBlocked + /* What the model on the other end can do with a picture, read at send time + rather than closed over: the send and edit-queued callbacks are built + before the catalog lookup below, and a model switched between typing and + sending has to be the one the guard judges. Filled in further down. */ + const visionRef = useRef<{ supports?: boolean; model: string | null }>({ + supports: undefined, + model: null, + }) + // Live view of the transcript for the long-running stream loop: the // session-events reconciler (use-conversations) may add/replace rows while // a turn is in flight, and dedupe-by-functionTriggerId must see them. @@ -580,17 +588,22 @@ export function ChatView({ ).blocks } } - // Same expansion as the live send path: a queued message's PDFs have - // to reach the agent as markdown too, or editing a queued message - // would silently drop the document it carried. + // Same expansion as the live send path: a queued message's documents + // and pictures have to reach the agent too, or editing a queued + // message would silently drop what it carried. + let attachedImages: HarnessImageBlock[] | undefined if ( backend.id === 'real' && - payload.attachments.some(isPdfAttachment) + hasExpandableAttachments(payload.attachments) ) { - const expanded = await expandPdfAttachments(payload.attachments) + const expanded = await expandAttachments(payload.attachments, { + vision: visionRef.current.supports, + model: visionRef.current.model, + }) if (expanded.blocks.length > 0) { attachedBlocks = [...(attachedBlocks ?? []), ...expanded.blocks] } + if (expanded.images.length > 0) attachedImages = expanded.images // Same reporting as the live send path. Staying silent here would let // an edited queued message lose its document with no explanation. for (const failure of expanded.failures) { @@ -608,7 +621,9 @@ export function ChatView({ conversationId, id, payload.text, - attachedBlocks ? { attachedBlocks } : undefined, + attachedBlocks || attachedImages + ? { attachedBlocks, attachedImages } + : undefined, ) } catch (err) { onAppendMessage( @@ -648,6 +663,16 @@ export function ChatView({ return match?.contextWindow }, [modelOptions, effectiveModel]) + /* What the send path may do with an attached picture. `undefined` when the + catalog has no row or the router said nothing — the attachment router + treats that as "send it", so a missing capability flag never silently + eats an image. */ + const modelVision = useMemo(() => { + const match = modelOptions.find((o) => o.id === effectiveModel) + return match?.supportsVision + }, [modelOptions, effectiveModel]) + visionRef.current = { supports: modelVision, model: effectiveModel } + /* Injected session chips (the `chat` extension slot), rendered in the * header's right cluster where the built-in context meter sits. A chip * with id `context` supersedes the estimate-based ContextUsage meter — @@ -1156,30 +1181,43 @@ export function ChatView({ } } - // A PDF is not text: read as bytes it reaches the model as noise, so the - // `pdf` worker converts it on this machine and the markdown is appended - // as another attachment block. Failures never block the send — an - // unreadable document becomes a placeholder block plus a warn notice, so - // the model knows it was handed something it could not read. - if (backend.id === 'real' && payload.attachments.some(isPdfAttachment)) { - const expanded = await expandPdfAttachments(payload.attachments) + // Attachments are not text. A PDF or an office document read as bytes + // reaches the model as noise, and an image reaches it as nothing at all, + // so each kind is expanded on this machine first: documents into + // `` markdown blocks, pictures into image content + // blocks. Failures never block the send — an unreadable attachment + // becomes a placeholder block plus a warn notice, so the model knows it + // was handed something that could not be read. + let attachedImages: HarnessImageBlock[] | undefined + if ( + backend.id === 'real' && + hasExpandableAttachments(payload.attachments) + ) { + const expanded = await expandAttachments(payload.attachments, { + vision: visionRef.current.supports, + model: visionRef.current.model, + }) if (expanded.blocks.length > 0) { attachedBlocks = [...(attachedBlocks ?? []), ...expanded.blocks] } - // Relabel the chip with what the worker made of the document. The - // expansion runs before the model is called, so it never shows up as a - // function call — without this a person has no way to tell the PDF was - // read at all. - if (expanded.read.length > 0 && !willQueue) { - const byId = new Map(expanded.read.map((r) => [r.id, r])) + if (expanded.images.length > 0) attachedImages = expanded.images + // Drop the source bytes and relabel the chip with what the expansion + // made of each attachment. The relabel runs before the model is called, + // so it never shows up as a function call — without it a person has no + // way to tell the document was read at all. + // + // The `file` removal is NOT conditional on anything having been read: + // an attachment that failed, or an image refused for a model that + // cannot see, has finished its job too, and keeping its bytes would + // hold the whole file in memory for as long as the conversation stays + // open. Only the label depends on a matching entry. + if (!willQueue) { + const byId = new Map(expanded.read.map((r) => [r.id, r.label])) onPatchMessage(conversationId, userMsg.id, { - // `file` is dropped here as well as relabelled. It has done its job - // by now, and keeping it would hold the whole document in memory - // for as long as the conversation stays open. attachments: (userMsg.attachments ?? []).map(({ file, ...a }) => { void file - const summary = byId.get(a.id) - return summary ? { ...a, name: summaryLabel(a.name, summary) } : a + const label = byId.get(a.id) + return label ? { ...a, name: label } : a }), }) } @@ -1216,6 +1254,9 @@ export function ChatView({ ...(attachedBlocks && attachedBlocks.length > 0 ? { attachedBlocks } : {}), + ...(attachedImages && attachedImages.length > 0 + ? { attachedImages } + : {}), }, ) } catch (err) { @@ -1264,6 +1305,9 @@ export function ChatView({ ...(attachedBlocks && attachedBlocks.length > 0 ? { attachedBlocks } : {}), + ...(attachedImages && attachedImages.length > 0 + ? { attachedImages } + : {}), }, )) { switch (event.kind) { diff --git a/console/web/src/components/chat/Composer.tsx b/console/web/src/components/chat/Composer.tsx index e54898ad2..757048c24 100644 --- a/console/web/src/components/chat/Composer.tsx +++ b/console/web/src/components/chat/Composer.tsx @@ -7,6 +7,7 @@ import { import { ArrowUp, Loader2, Square } from 'lucide-react' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { PermissionModePicker } from '@/components/permissions/PermissionModePicker' +import { attachmentsFromFiles } from '@/lib/attachments/from-files' import type { PermissionMode } from '@/lib/backend/approval-settings' import type { FunctionEntry } from '@/lib/functions' import { cn } from '@/lib/utils' @@ -25,6 +26,7 @@ import { LexicalShell } from './LexicalShell' import { ModelPicker } from './ModelPicker' import { ModePicker } from './ModePicker' import { nextHistoryTarget } from './queue-history' +import { useFileDrop } from './use-file-drop' export interface ComposerSubmitPayload { text: string @@ -297,8 +299,38 @@ export function Composer({ setAttachments((current) => current.filter((a) => a.id !== id)) }, []) + const attachFiles = useCallback( + async (files: File[]) => { + if (files.length === 0) return + handleAttach(await attachmentsFromFiles(files)) + }, + [handleAttach], + ) + + // The drop zone is the whole chat pane, claimed in the capture phase — see + // `use-file-drop`. A drop onto the transcript, where people actually let go + // of a screenshot, lands here too, and the editor never gets to eat it. + const shell = useRef(null) + const dragging = useFileDrop({ + anchorRef: shell, + disabled: Boolean(inputDisabled), + onFiles: (files) => void attachFiles(files), + }) + return ( -
+
+ {dragging ? ( +
+ drop to attach +
+ ) : null} + {attachments.length > 0 ? (
{attachments.map((a) => ( diff --git a/console/web/src/components/chat/use-file-drop.ts b/console/web/src/components/chat/use-file-drop.ts new file mode 100644 index 000000000..69f4149de --- /dev/null +++ b/console/web/src/components/chat/use-file-drop.ts @@ -0,0 +1,143 @@ +/** + * Dropping a file anywhere on a chat, and pasting one into it. + * + * Two things make this harder than an `onDrop` prop, and both were live + * failures before this hook existed. + * + * The editor eats the event. A drop lands on the innermost element under the + * cursor, which over the composer is Lexical's `contenteditable`, and its + * plain-text plugin handles `dragover`/`drop`/`paste` on that element. React + * props on an ancestor run afterwards, if at all. So the listeners here are + * NATIVE and CAPTURE-phase: capture runs root to target, so a file drop is + * claimed and stopped before the editor ever sees it. Text drags are left + * alone — dragging a word out of the transcript into the composer still works. + * + * The target is the whole conversation, not the composer box. People drop a + * screenshot onto the message they are reading, which is the transcript, a + * hundred pixels above the box. Scoping the zone to the composer meant most + * real drops hit nothing at all, which reads as "drag and drop does not work". + * The zone is the enclosing chat pane, found from the composer's own node, so + * two chats side by side keep separate zones and a worker page's own drop zone + * is untouched. + */ + +import { type RefObject, useEffect, useRef, useState } from 'react' + +/** Marks the chat pane. `ChatView` puts this on its root `
`. */ +const CHAT_PANE_SELECTOR = '[data-chat-session-id]' + +/** + * `true` when a drag carries files rather than selected text. During a drag + * the browser withholds the data itself, so `types` is the only thing to read. + */ +function carriesFiles(e: DragEvent): boolean { + // `types` is already a readonly array of strings, and this runs on every + // `dragover` for the whole gesture, so there is nothing to copy it into. + return (e.dataTransfer?.types ?? []).includes('Files') +} + +interface FileDropOptions { + /** Any node inside the pane that should accept drops. */ + anchorRef: RefObject + /** No attaching while the composer is blocked or a turn is locked. */ + disabled: boolean + onFiles: (files: File[]) => void +} + +/** `true` while a file drag is over the pane, for the drop affordance. */ +export function useFileDrop({ + anchorRef, + disabled, + onFiles, +}: FileDropOptions): boolean { + const [dragging, setDragging] = useState(false) + + // Read through refs so a re-render never rebinds the listeners: rebinding + // mid-drag drops the depth count and the highlight sticks on. + const disabledRef = useRef(disabled) + disabledRef.current = disabled + const onFilesRef = useRef(onFiles) + onFilesRef.current = onFiles + + useEffect(() => { + const anchor = anchorRef.current + if (!anchor) return + const host = anchor.closest(CHAT_PANE_SELECTOR) ?? anchor + + // Depth counter, not a boolean: dragging across a child fires `dragleave` + // on the element being left before `dragenter` on the one being entered, + // so a boolean flickers the highlight off over every control in the pane. + let depth = 0 + + // A file drag is consumed whether or not the composer will accept it. + // Returning early while disabled leaves the browser's own handling in + // place, and the browser's handling of a dropped PDF is to navigate to it: + // the console is replaced by a document viewer and the conversation is + // gone. Disabled means "do not attach", not "let the page eat itself". + const handleDragEnter = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + if (disabledRef.current) return + depth += 1 + setDragging(true) + } + + const handleDragOver = (e: DragEvent) => { + if (!carriesFiles(e)) return + // Without a prevented `dragover` the browser never fires `drop` at all. + e.preventDefault() + e.stopPropagation() + if (disabledRef.current) return + if (e.dataTransfer) e.dataTransfer.dropEffect = 'copy' + } + + const handleDragLeave = (e: DragEvent) => { + if (!carriesFiles(e)) return + depth = Math.max(0, depth - 1) + if (depth === 0) setDragging(false) + } + + const handleDrop = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + e.stopPropagation() + depth = 0 + setDragging(false) + if (disabledRef.current) return + const files = Array.from(e.dataTransfer?.files ?? []) + if (files.length > 0) onFilesRef.current(files) + } + + // A screenshot pasted from the clipboard carries no text, and a file copied + // from a file manager carries only its name as text — which would land in + // the message beside its own chip. Consume the paste in both cases. + const handlePaste = (e: ClipboardEvent) => { + const files = Array.from(e.clipboardData?.files ?? []) + if (files.length === 0) return + e.preventDefault() + e.stopPropagation() + if (disabledRef.current) return + onFilesRef.current(files) + } + + // `host` is an `Element`, whose overloads type every listener as taking a + // bare `Event`; the drag and clipboard events are narrowed above. + const bindings: Array<[string, EventListener]> = [ + ['dragenter', handleDragEnter as EventListener], + ['dragover', handleDragOver as EventListener], + ['dragleave', handleDragLeave as EventListener], + ['drop', handleDrop as EventListener], + ['paste', handlePaste as EventListener], + ] + for (const [type, listener] of bindings) { + host.addEventListener(type, listener, true) + } + return () => { + for (const [type, listener] of bindings) { + host.removeEventListener(type, listener, true) + } + } + }, [anchorRef]) + + return dragging && !disabled +} diff --git a/console/web/src/lib/attachments/documents.test.ts b/console/web/src/lib/attachments/documents.test.ts new file mode 100644 index 000000000..a8c013400 --- /dev/null +++ b/console/web/src/lib/attachments/documents.test.ts @@ -0,0 +1,215 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { Attachment } from '@/types/chat' +import { + expandDocumentAttachments, + isDocumentAttachment, + MAX_DOCUMENT_BYTES, + MAX_DOCUMENTS_PER_SEND, + TO_MARKDOWN_FUNCTION_ID, +} from './documents' + +function doc(name = 'report.docx', bytes = 'PK'): Attachment { + return { + id: name, + name, + size: bytes.length, + type: '', + file: new File([bytes], name), + } +} + +function converted(over: Record = {}) { + return { + format: 'docx', + family: 'prose', + detected_from: 'content', + body: { + text: '# Quarterly Notes', + chars: 17, + total_chars: 17, + truncated: false, + }, + asset_count: 0, + elapsed_ms: 4, + ...over, + } +} + +describe('isDocumentAttachment', () => { + it('recognises every office format by extension', () => { + for (const name of [ + 'a.docx', + 'a.doc', + 'a.pptx', + 'a.xlsx', + 'a.xlsb', + 'a.odt', + 'a.ods', + 'a.odp', + 'a.rtf', + 'a.epub', + 'a.csv', + ]) { + expect(isDocumentAttachment(doc(name)), name).toBe(true) + } + }) + + /* The browser's MIME type for a CSV is `application/vnd.ms-excel`, and for a + file dragged out of an archive it is often nothing at all — the name is the + one thing that survives every route into the composer. */ + it('ignores the declared MIME type', () => { + const csv: Attachment = { + ...doc('rows.csv'), + type: 'application/vnd.ms-excel', + } + expect(isDocumentAttachment(csv)).toBe(true) + }) + + it('leaves PDFs and images to their own paths', () => { + expect(isDocumentAttachment(doc('report.pdf'))).toBe(false) + expect(isDocumentAttachment(doc('shot.png'))).toBe(false) + }) +}) + +describe('expandDocumentAttachments', () => { + it('does nothing when there is no document', async () => { + const trigger = vi.fn() + const result = await expandDocumentAttachments([doc('shot.png')], trigger) + expect(result.blocks).toEqual([]) + expect(trigger).not.toHaveBeenCalled() + }) + + /* A CSV has no signature of its own. Without the name the worker cannot + recognise it and refuses a file it reads perfectly well. */ + it('sends the file name alongside the bytes', async () => { + const trigger = vi.fn().mockResolvedValue(converted({ format: 'csv' })) + await expandDocumentAttachments([doc('rows.csv', 'a,b\n')], trigger) + + expect(trigger).toHaveBeenCalledWith( + TO_MARKDOWN_FUNCTION_ID, + expect.objectContaining({ file_name: 'rows.csv' }), + ) + }) + + it('wraps the markdown in an attached-file block', async () => { + const trigger = vi.fn().mockResolvedValue(converted()) + const result = await expandDocumentAttachments([doc()], trigger) + + expect(result.blocks).toHaveLength(1) + expect(result.blocks[0]).toContain('path="report.docx"') + expect(result.blocks[0]).toContain('format="docx-markdown"') + expect(result.blocks[0]).toContain('# Quarterly Notes') + expect(result.failures).toEqual([]) + expect(result.read[0].label).toContain('docx') + }) + + /* Markdown renders an embedded image as alt text, so a deck of diagrams + converts to almost nothing. The block has to say the pictures exist, or the + model reports an empty document. */ + it('names the images the markdown could not carry', async () => { + const trigger = vi.fn().mockResolvedValue( + converted({ + format: 'pptx', + asset_count: 12, + body: { text: 'Roadmap', chars: 7, total_chars: 7, truncated: false }, + }), + ) + const result = await expandDocumentAttachments([doc('deck.pptx')], trigger) + + expect(result.blocks[0]).toContain('embedded-images="12"') + expect(result.blocks[0]).toContain('document::extract-assets') + expect(result.read[0].label).toContain('12 images') + }) + + /* An empty conversion and a deck whose content is pictures look identical on + the wire. The block is the only place that difference can be stated. */ + it('distinguishes an image-only document from an empty one', async () => { + const pictures = vi.fn().mockResolvedValue( + converted({ + asset_count: 3, + body: { text: '', chars: 0, total_chars: 0, truncated: false }, + }), + ) + const withPictures = await expandDocumentAttachments( + [doc('deck.pptx')], + pictures, + ) + expect(withPictures.blocks[0]).toContain('NOT included in this message') + expect(withPictures.blocks[0]).toContain('Do not call the document empty') + + const blank = vi.fn().mockResolvedValue( + converted({ + asset_count: 0, + body: { text: '', chars: 0, total_chars: 0, truncated: false }, + }), + ) + const withNothing = await expandDocumentAttachments([doc()], blank) + expect(withNothing.blocks[0]).toContain('no text and no images') + }) + + it('reports truncation with the way to get the rest', async () => { + const trigger = vi.fn().mockResolvedValue( + converted({ + body: { + text: 'start', + chars: 5, + total_chars: 90_000, + truncated: true, + }, + }), + ) + const result = await expandDocumentAttachments([doc()], trigger) + + expect(result.blocks[0]).toContain('truncated="true"') + expect(result.blocks[0]).toContain('total-chars="90000"') + expect(result.blocks[0]).toContain('max_chars 0') + expect(result.read[0].label).toContain('90,000+ chars') + }) + + /* A failure has to reach the model as a block. Staying silent is the bug this + path exists to prevent: the agent answers as though nothing was attached. */ + it('turns a worker failure into a block and a named failure', async () => { + const trigger = vi + .fn() + .mockRejectedValue(new Error('function document::to-markdown not found')) + const result = await expandDocumentAttachments([doc()], trigger) + + expect(result.blocks[0]).toContain('error=') + expect(result.blocks[0]).toContain('iii worker add document') + expect(result.failures).toHaveLength(1) + }) + + it('refuses a document over the composer ceiling without calling the worker', async () => { + const trigger = vi.fn() + const huge: Attachment = { ...doc(), size: MAX_DOCUMENT_BYTES + 1 } + const result = await expandDocumentAttachments([huge], trigger) + + expect(trigger).not.toHaveBeenCalled() + expect(result.failures[0].reason).toContain('limit') + }) + + it('reports the documents past the per-send ceiling instead of dropping them', async () => { + const trigger = vi.fn().mockResolvedValue(converted()) + const many = Array.from({ length: MAX_DOCUMENTS_PER_SEND + 2 }, (_, i) => + doc(`report-${i}.docx`), + ) + const result = await expandDocumentAttachments(many, trigger) + + expect(trigger).toHaveBeenCalledTimes(MAX_DOCUMENTS_PER_SEND) + expect(result.blocks).toHaveLength(MAX_DOCUMENTS_PER_SEND + 2) + expect(result.failures).toHaveLength(2) + expect(result.failures[0].reason).toContain('per message') + }) + + /* A conversation reloaded from history keeps the chip, not the bytes. */ + it('skips an attachment with no file', async () => { + const trigger = vi.fn() + const { file, ...withoutBytes } = doc() + void file + const result = await expandDocumentAttachments([withoutBytes], trigger) + + expect(result.blocks).toEqual([]) + expect(trigger).not.toHaveBeenCalled() + }) +}) diff --git a/console/web/src/lib/attachments/documents.ts b/console/web/src/lib/attachments/documents.ts new file mode 100644 index 000000000..e10eab4a4 --- /dev/null +++ b/console/web/src/lib/attachments/documents.ts @@ -0,0 +1,269 @@ +/** + * Office-document expansion for the composer send path. + * + * A `.docx` or a `.pptx` attached in the composer used to reach the agent as + * nothing at all: the send path forwards text blocks, and a ZIP of XML is not + * text. The agent then answered as though no document had been given to it — + * the same hole the PDF path closed, for every other format people attach. + * + * At send time each one is converted by the `document` worker on the machine + * and appended as an `` text block, the same envelope + * `#file()` mentions and PDFs already use. + * + * Conversion is one call, not two. Unlike a PDF there is no classification + * step: an office document either parses or it does not, and the format is + * read from the bytes. What the block does carry is the count of images the + * markdown could not represent, because a deck built out of diagrams converts + * to a page of titles and would otherwise read as a document with little to + * say. + */ + +import type { Attachment } from '@/types/chat' +import { + ATTACHED_FILE_PREFIX, + type AttachmentFailure, + type AttachmentReadSummary, + describeWorkerFailure, + escapeAttr, + extensionOf, + failureBlock, + fileToBase64, + reportDropped, + type TriggerFn, + triggerOr, +} from './shared' + +export const TO_MARKDOWN_FUNCTION_ID = 'document::to-markdown' + +/** Max documents converted per send; extras are reported, never dropped. */ +export const MAX_DOCUMENTS_PER_SEND = 4 + +/** + * Characters of markdown inlined per document. A long report would otherwise + * consume the context the question needed. The block says when it stops short, + * and the agent can call `document::to-markdown` itself for the rest. + */ +export const MAX_MARKDOWN_CHARS = 20_000 + +/** + * Largest document read from the composer. Encoding happens in the browser, so + * an enormous file is a frozen tab before the worker ever sees it, and the + * worker's own ceiling would reject it anyway. Refuse it here with an + * explanation instead. + */ +export const MAX_DOCUMENT_BYTES = 64 * 1024 * 1024 + +/** + * Every extension the `document` worker converts, minus `pdf`, which has its + * own worker and its own path through the send. + * + * Extension rather than MIME type on purpose: browsers report office documents + * inconsistently (a `.csv` arrives as `application/vnd.ms-excel`, a file + * dragged from an archive often arrives as `application/octet-stream` or with + * no type at all), and the name is the one thing that survives every route into + * the composer. + */ +export const DOCUMENT_EXTENSIONS = new Set([ + 'doc', + 'docx', + 'docm', + 'ppt', + 'pps', + 'pot', + 'pptx', + 'pptm', + 'ppsx', + 'ppsm', + 'xls', + 'xlsx', + 'xlsm', + 'xlsb', + 'odt', + 'ods', + 'odp', + 'rtf', + 'epub', + 'csv', +]) + +export interface ExpandedDocuments { + /** One `` block per document, in input order. */ + blocks: string[] + /** One entry per document actually converted, for the message chips. */ + read: AttachmentReadSummary[] + failures: AttachmentFailure[] +} + +/** Whether an attachment is an office document this worker converts. */ +export function isDocumentAttachment(attachment: Attachment): boolean { + return DOCUMENT_EXTENSIONS.has(extensionOf(attachment.name)) +} + +// --- wire subset of the document worker ----------------------------------- + +interface MarkdownWire { + format?: string + family?: string + detected_from?: 'requested' | 'content' | 'extension' + body?: { + text?: string + chars?: number + total_chars?: number + truncated?: boolean + } + asset_count?: number + elapsed_ms?: number +} + +/** + * Convert every attached office document through the `document` worker. + * + * Attachments without their underlying `File` are skipped silently: a + * conversation reloaded from history carries the chip metadata but not the + * bytes, and re-reading a document attached in a previous session is not this + * function's job. + */ +export async function expandDocumentAttachments( + attachments: Attachment[], + trigger?: TriggerFn, +): Promise { + const documents = attachments.filter((a) => isDocumentAttachment(a) && a.file) + if (documents.length === 0) return { blocks: [], read: [], failures: [] } + + const call = triggerOr(trigger) + + const blocks: string[] = [] + const read: AttachmentReadSummary[] = [] + const failures: AttachmentFailure[] = [] + + for (const attachment of documents.slice(0, MAX_DOCUMENTS_PER_SEND)) { + if (attachment.size > MAX_DOCUMENT_BYTES) { + const mb = Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024)) + const reason = `larger than the ${mb} MB limit for reading a document in the composer` + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + continue + } + try { + const outcome = await expandOne(attachment, call) + blocks.push(outcome.block) + read.push(outcome.summary) + } catch (err) { + const reason = describeWorkerFailure(err, 'document') + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + } + } + + reportDropped( + documents.slice(MAX_DOCUMENTS_PER_SEND), + `only ${MAX_DOCUMENTS_PER_SEND} documents are read per message`, + { blocks, failures }, + ) + + return { blocks, read, failures } +} + +interface ExpandOutcome { + block: string + summary: AttachmentReadSummary +} + +async function expandOne( + attachment: Attachment, + call: TriggerFn, +): Promise { + const bytes_base64 = await fileToBase64(attachment.file as File) + + const converted = (await call(TO_MARKDOWN_FUNCTION_ID, { + bytes_base64, + // A CSV carries no signature of its own; without the name the worker + // cannot recognise it and would refuse a file it reads perfectly well. + file_name: attachment.name, + max_chars: MAX_MARKDOWN_CHARS, + })) as MarkdownWire + + const body = converted.body ?? {} + const text = body.text ?? '' + const assets = converted.asset_count ?? 0 + const elapsedMs = converted.elapsed_ms ?? 0 + const chars = body.total_chars ?? text.length + + const attrs = [ + `path="${escapeAttr(attachment.name)}"`, + `size="${attachment.size}"`, + `format="${escapeAttr(converted.format ?? extensionOf(attachment.name))}-markdown"`, + ] + if (body.truncated) { + attrs.push('truncated="true"') + attrs.push(`total-chars="${chars}"`) + } + if (assets > 0) attrs.push(`embedded-images="${assets}"`) + + const notes: string[] = [] + if (body.truncated) { + notes.push( + `This is the first ${body.chars ?? text.length} of ${chars} characters. Call ${TO_MARKDOWN_FUNCTION_ID} with max_chars 0 for the rest.`, + ) + } + if (assets > 0) { + notes.push( + `${assets} embedded image${assets === 1 ? '' : 's'} could not be represented as markdown. Call document::extract-assets for their bytes.`, + ) + } + // The empty conversion is the case worth spelling out. A deck of diagrams + // and a genuinely blank file both come back with no text, and the model has + // no way to tell them apart unless the block says which happened. + if (text.trim().length === 0) { + notes.push( + assets > 0 + ? `This document holds no text an agent can read: its content is ${assets} embedded image${assets === 1 ? '' : 's'}, which are NOT included in this message. Call document::extract-assets for them, or document::ocr to have them transcribed. Do not call the document empty.` + : 'This document converted to nothing: it holds no text and no images.', + ) + } + + const preamble = notes.length > 0 ? `${notes.join(' ')}\n\n` : '' + return { + block: `${ATTACHED_FILE_PREFIX}${attrs.join(' ')}>\n${preamble}${text}\n`, + summary: { + id: attachment.id, + label: chipLabel(attachment.name, { + format: converted.format, + chars, + truncated: body.truncated === true, + assets, + elapsedMs, + }), + }, + } +} + +/** + * One line for the chip on the sent message: what the worker made of the + * document, and how fast. This is the only place a person can see that the + * document was read at all, because the conversion happens before the model is + * called and so never appears as a function call in the transcript. + */ +function chipLabel( + name: string, + summary: { + format?: string + chars: number + truncated: boolean + assets: number + elapsedMs: number + }, +): string { + const parts: string[] = [] + if (summary.format) parts.push(summary.format) + parts.push( + summary.chars > 0 + ? `${summary.chars.toLocaleString('en-US')}${summary.truncated ? '+' : ''} chars` + : 'no text', + ) + if (summary.assets > 0) { + parts.push(`${summary.assets} image${summary.assets === 1 ? '' : 's'}`) + } + parts.push(`${summary.elapsedMs} ms`) + return `${name} · ${parts.join(' · ')}` +} diff --git a/console/web/src/lib/attachments/from-files.ts b/console/web/src/lib/attachments/from-files.ts new file mode 100644 index 000000000..f85840055 --- /dev/null +++ b/console/web/src/lib/attachments/from-files.ts @@ -0,0 +1,63 @@ +/** + * One file → one attachment, however it arrived. + * + * The composer takes files three ways — the paperclip, a drag onto the panel, + * a paste — and all three have to produce the same thing, or a screenshot + * pasted in behaves differently from the same screenshot picked from a dialog. + */ + +import { uid } from '@/hooks/use-conversations' +import type { Attachment } from '@/types/chat' + +/** Largest file read for a chip preview. Beyond this the chip shows an icon. */ +const MAX_PREVIEW_BYTES = 1_000_000 + +/** + * A data URL for the chip, for the two kinds where it is worth having: a + * thumbnail of an image, and the first bytes of a text file. Never fails a + * pick — a preview that cannot be read is simply absent. + */ +function readPreview(file: File): Promise { + if (file.size > MAX_PREVIEW_BYTES) return Promise.resolve(undefined) + if (!/^(image|text)\//.test(file.type)) return Promise.resolve(undefined) + return new Promise((resolve) => { + const reader = new FileReader() + reader.onload = () => + resolve(typeof reader.result === 'string' ? reader.result : undefined) + reader.onerror = () => resolve(undefined) + reader.readAsDataURL(file) + }) +} + +/** + * Build attachments for a set of picked, dropped or pasted files. + * + * The `File` is kept on the attachment so the send path can hand the bytes to + * whichever worker reads that kind. It is browser-only and never persisted: a + * conversation reloaded from history keeps the chip, not the document. + */ +export async function attachmentsFromFiles( + files: File[], +): Promise { + return Promise.all( + files.map(async (file) => ({ + id: uid(), + name: nameOf(file), + size: file.size, + type: file.type || 'application/octet-stream', + dataUrl: await readPreview(file), + file, + })), + ) +} + +/** + * A pasted screenshot arrives as `image.png` on every platform, which makes + * three of them indistinguishable in the chip strip. Only a genuinely nameless + * file gets a generated name; a real one keeps its own. + */ +function nameOf(file: File): string { + if (file.name) return file.name + const extension = file.type.split('/')[1] ?? 'bin' + return `pasted.${extension}` +} diff --git a/console/web/src/lib/attachments/images.test.ts b/console/web/src/lib/attachments/images.test.ts new file mode 100644 index 000000000..f6ad20fa9 --- /dev/null +++ b/console/web/src/lib/attachments/images.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { Attachment } from '@/types/chat' +import { + dimensionsOf, + exceedsEdge, + expandImageAttachments, + fitWithin, + imageMimeOf, + isImageAttachment, + MAX_IMAGE_BYTES, + MAX_IMAGE_EDGE, + MAX_IMAGES_PER_SEND, + MAX_SOURCE_IMAGE_BYTES, + needsDownscale, +} from './images' + +function image(name = 'shot.png', type = 'image/png', size = 1024): Attachment { + const file = new File([new Uint8Array(size)], name, { type }) + return { id: name, name, size, type, file } +} + +/** A file that reports a size without allocating it. */ +function oversized(name: string, type: string, size: number): Attachment { + const attachment = image(name, type, 8) + Object.defineProperty(attachment.file as File, 'size', { value: size }) + return { ...attachment, size } +} + +describe('isImageAttachment', () => { + it('takes the declared type first and the extension second', () => { + expect(isImageAttachment(image())).toBe(true) + expect(isImageAttachment(image('photo.HEIC', ''))).toBe(true) + expect(isImageAttachment(image('notes.txt', 'text/plain'))).toBe(false) + }) +}) + +describe('imageMimeOf', () => { + it('normalises jpg to the type providers expect', () => { + expect(imageMimeOf(image('a.jpg', ''))).toBe('image/jpeg') + expect(imageMimeOf(image('a.png', 'image/png'))).toBe('image/png') + }) +}) + +describe('fitWithin', () => { + it('keeps the aspect ratio and caps the longest edge', () => { + expect(fitWithin(3200, 1600)).toEqual({ + width: MAX_IMAGE_EDGE, + height: MAX_IMAGE_EDGE / 2, + }) + }) + + /* Upscaling a small screenshot would add bytes and no detail. */ + it('leaves an image inside the ceiling alone', () => { + expect(fitWithin(800, 600)).toEqual({ width: 800, height: 600 }) + }) +}) + +/** A PNG header carrying the given dimensions and nothing else of substance. */ +function pngHeader(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(new ArrayBuffer(32)) + bytes.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + const be = (at: number, value: number) => { + bytes[at] = (value >>> 24) & 0xff + bytes[at + 1] = (value >>> 16) & 0xff + bytes[at + 2] = (value >>> 8) & 0xff + bytes[at + 3] = value & 0xff + } + be(16, width) + be(20, height) + return bytes +} + +describe('dimensionsOf', () => { + it('reads a PNG header without decoding the image', () => { + expect(dimensionsOf(pngHeader(8000, 1200))).toEqual({ + width: 8000, + height: 1200, + }) + }) + + it('reads a JPEG start-of-frame past its other segments', () => { + // SOI, an APP0 segment to skip, then SOF0 carrying 4000x3000. + const bytes = new Uint8Array([ + 0xff, 0xd8, 0xff, 0xe0, 0x00, 0x04, 0x00, 0x00, 0xff, 0xc0, 0x00, 0x11, + 0x08, 0x0b, 0xb8, 0x0f, 0xa0, 0x03, 0x01, 0x22, 0x00, + ]) + expect(dimensionsOf(bytes)).toEqual({ width: 4000, height: 3000 }) + }) + + it('says nothing for a format it does not parse', () => { + expect(dimensionsOf(new Uint8Array([0x00, 0x01, 0x02, 0x03]))).toBeNull() + }) +}) + +describe('exceedsEdge', () => { + /* Bytes are a poor proxy for pixels: a flat-coloured screenshot compresses to + almost nothing at eight thousand pixels wide, and every one of those pixels + is billed. */ + it('catches a tiny file with enormous dimensions', () => { + expect(exceedsEdge(pngHeader(8000, 1200))).toBe(true) + expect(exceedsEdge(pngHeader(1200, 800))).toBe(false) + }) +}) + +describe('needsDownscale', () => { + it('triggers on the byte ceiling', () => { + expect(needsDownscale({ size: MAX_IMAGE_BYTES + 1 })).toBe(true) + expect(needsDownscale({ size: 1024 })).toBe(false) + }) +}) + +describe('expandImageAttachments', () => { + const neverDownscales = vi.fn().mockResolvedValue(null) + + it('sends a supported image as a native image block', async () => { + const result = await expandImageAttachments([image()], neverDownscales) + + expect(result.images).toHaveLength(1) + expect(result.images[0].type).toBe('image') + expect(result.images[0].mime).toBe('image/png') + expect(result.images[0].data.length).toBeGreaterThan(0) + expect(result.blocks).toEqual([]) + expect(result.failures).toEqual([]) + }) + + /* A `.heic` from a phone is a file no provider decodes. A block saying so + beats an API error the person never sees. */ + it('refuses a format no model reads when it cannot be converted', async () => { + const result = await expandImageAttachments( + [image('photo.heic', 'image/heic')], + neverDownscales, + ) + + expect(result.images).toEqual([]) + expect(result.blocks[0]).toContain('not a format a model can read') + expect(result.failures).toHaveLength(1) + }) + + it('converts an unsupported format when the browser can', async () => { + const downscale = vi.fn().mockResolvedValue({ + blob: new Blob([new Uint8Array(64)]), + mime: 'image/jpeg', + }) + const result = await expandImageAttachments( + [image('photo.heic', 'image/heic')], + downscale, + ) + + expect(downscale).toHaveBeenCalled() + expect(result.images[0].mime).toBe('image/jpeg') + expect(result.read[0].label).toContain('resized') + }) + + /* The regression this closes: a highly compressed image sailed under the + byte ceiling and was sent at full resolution. */ + it('downscales a small file whose pixels are over the edge ceiling', async () => { + const downscale = vi.fn().mockResolvedValue({ + blob: new Blob([new Uint8Array(32)]), + mime: 'image/jpeg', + }) + const header = pngHeader(8000, 1200) + const attachment: Attachment = { + id: 'wide.png', + name: 'wide.png', + size: header.length, + type: 'image/png', + file: new File([header], 'wide.png', { type: 'image/png' }), + } + const result = await expandImageAttachments([attachment], downscale) + + expect(downscale).toHaveBeenCalledTimes(1) + expect(result.images).toHaveLength(1) + expect(result.read[0].label).toContain('resized') + }) + + it('downscales an image over the byte ceiling', async () => { + const downscale = vi.fn().mockResolvedValue({ + blob: new Blob([new Uint8Array(32)]), + mime: 'image/jpeg', + }) + const big = oversized('screenshot.png', 'image/png', MAX_IMAGE_BYTES + 1) + const result = await expandImageAttachments([big], downscale) + + expect(downscale).toHaveBeenCalledTimes(1) + expect(result.images).toHaveLength(1) + expect(result.read[0].label).toContain('resized') + }) + + it('reports an oversized image the browser could not resize', async () => { + const big = oversized('screenshot.png', 'image/png', MAX_IMAGE_BYTES + 1) + const result = await expandImageAttachments([big], neverDownscales) + + expect(result.images).toEqual([]) + expect(result.blocks[0]).toContain('could not be resized') + expect(result.failures).toHaveLength(1) + }) + + /* Past this size the browser stalls the tab decoding it, so nothing is even + attempted. */ + it('refuses an enormous source file outright', async () => { + const downscale = vi.fn() + const enormous = oversized( + 'raw.png', + 'image/png', + MAX_SOURCE_IMAGE_BYTES + 1, + ) + const result = await expandImageAttachments([enormous], downscale) + + expect(downscale).not.toHaveBeenCalled() + expect(result.blocks[0]).toContain('too large to send') + }) + + it('reports the images past the per-send ceiling instead of dropping them', async () => { + const many = Array.from({ length: MAX_IMAGES_PER_SEND + 1 }, (_, i) => + image(`shot-${i}.png`), + ) + const result = await expandImageAttachments(many, neverDownscales) + + expect(result.images).toHaveLength(MAX_IMAGES_PER_SEND) + expect(result.failures).toHaveLength(1) + expect(result.failures[0].reason).toContain('per message') + }) +}) diff --git a/console/web/src/lib/attachments/images.ts b/console/web/src/lib/attachments/images.ts new file mode 100644 index 000000000..d11576f3e --- /dev/null +++ b/console/web/src/lib/attachments/images.ts @@ -0,0 +1,307 @@ +/** + * Image attachments, as pictures rather than as prose about pictures. + * + * An image pasted or dropped into the composer used to reach the model as + * nothing: the chip rendered a thumbnail, and the send forwarded text blocks + * only. Everything underneath was already in place — the harness carries + * `ContentBlock::Image`, and the Anthropic and OpenAI providers both map it — + * so the picture was being dropped one layer above the plumbing that could + * have delivered it. + * + * Two things happen here before an image goes out. It is checked against the + * formats every vision model accepts, because a `.heic` from a phone is a file + * no provider will decode and a block saying so is more useful than an API + * error. And it is downscaled when it is larger than a model can take: a + * screenshot on a retina display is routinely eight megabytes, and no answer + * gets better for the pixels beyond the long-edge ceiling. + */ + +import type { Attachment } from '@/types/chat' +import { + type AttachmentFailure, + type AttachmentImageBlock, + type AttachmentReadSummary, + extensionOf, + failureBlock, + fileToBase64, + formatBytes, + reportDropped, +} from './shared' + +/** Images sent per message. Each one costs real tokens on arrival. */ +export const MAX_IMAGES_PER_SEND = 8 + +/** + * The formats every vision provider decodes. Anything else is refused by name + * rather than sent and rejected by the API. + */ +export const SUPPORTED_IMAGE_MIME = new Set([ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', +]) + +/** + * Longest edge kept when an image is downscaled. Beyond roughly this size the + * models resize server-side anyway, so the extra pixels only cost upload time + * and tokens. + */ +export const MAX_IMAGE_EDGE = 1568 + +/** + * Bytes above which an image is downscaled before sending. Providers cap the + * encoded payload at around five megabytes, and base64 inflates by a third, so + * the ceiling here is on the raw file. + */ +export const MAX_IMAGE_BYTES = 3.5 * 1024 * 1024 + +/** + * Hard refusal ceiling. Past this the browser is decoding a file large enough + * to stall the tab, and the answer is to point at the file rather than paste + * it. + */ +export const MAX_SOURCE_IMAGE_BYTES = 32 * 1024 * 1024 + +export interface ExpandedImages { + /** Native image content blocks for the outgoing message. */ + images: AttachmentImageBlock[] + /** Blocks explaining any image that could NOT be sent as a picture. */ + blocks: string[] + read: AttachmentReadSummary[] + failures: AttachmentFailure[] +} + +/** Whether an attachment is an image, by declared type or by extension. */ +export function isImageAttachment(attachment: Attachment): boolean { + if (attachment.type.startsWith('image/')) return true + return IMAGE_EXTENSIONS.has(extensionOf(attachment.name)) +} + +const IMAGE_EXTENSIONS = new Set([ + 'png', + 'jpg', + 'jpeg', + 'gif', + 'webp', + 'heic', + 'heif', + 'bmp', + 'tif', + 'tiff', + 'avif', +]) + +/** The MIME type to send under, from the declared type or the extension. */ +export function imageMimeOf(attachment: Attachment): string { + if (attachment.type.startsWith('image/')) return attachment.type + const ext = extensionOf(attachment.name) + if (ext === 'jpg' || ext === 'jpeg') return 'image/jpeg' + return ext ? `image/${ext}` : 'application/octet-stream' +} + +/** + * The dimensions an image is downscaled to: the same aspect ratio, longest + * edge at the ceiling. An image already inside the ceiling keeps its size — + * upscaling a small screenshot would add bytes and no detail. + */ +export function fitWithin( + width: number, + height: number, + edge = MAX_IMAGE_EDGE, +): { width: number; height: number } { + const longest = Math.max(width, height) + if (longest <= edge) return { width, height } + const scale = edge / longest + return { + width: Math.max(1, Math.round(width * scale)), + height: Math.max(1, Math.round(height * scale)), + } +} + +/** Whether this file has to be re-encoded before it can be sent. */ +export function needsDownscale(file: { size: number }): boolean { + return file.size > MAX_IMAGE_BYTES +} + +/** + * The pixel dimensions in an image's own header, without decoding it. + * + * Bytes are a poor proxy for size: a screenshot of a mostly-flat UI compresses + * to a few hundred kilobytes at eight thousand pixels wide, sails under the + * byte ceiling, and then costs a fortune in tokens for detail no model uses. + * Reading the header is cheap enough to do for every image and needs no canvas, + * which keeps the decision testable. + * + * `null` for a format not parsed here — the byte ceiling stays the backstop. + */ +export function dimensionsOf( + bytes: Uint8Array, +): { width: number; height: number } | null { + // Read the integers by hand rather than through a DataView: a Uint8Array's + // buffer may be a SharedArrayBuffer as far as the types are concerned, and + // these are four fields in three formats. + const u16 = (at: number) => (bytes[at] << 8) | bytes[at + 1] + const u16le = (at: number) => bytes[at] | (bytes[at + 1] << 8) + const u32 = (at: number) => + bytes[at] * 0x1000000 + + ((bytes[at + 1] << 16) | (bytes[at + 2] << 8) | bytes[at + 3]) + + // PNG: IHDR is always the first chunk, width and height at a fixed offset. + if (bytes.length > 24 && bytes[0] === 0x89 && bytes[1] === 0x50) { + return { width: u32(16), height: u32(20) } + } + + // GIF: little-endian, straight after the signature. + if (bytes.length > 10 && bytes[0] === 0x47 && bytes[1] === 0x49) { + return { width: u16le(6), height: u16le(8) } + } + + // JPEG: walk the segment chain to the start-of-frame, which is the only + // marker carrying the dimensions. + if (bytes.length > 4 && bytes[0] === 0xff && bytes[1] === 0xd8) { + let offset = 2 + while (offset + 9 < bytes.length) { + if (bytes[offset] !== 0xff) return null + const marker = bytes[offset + 1] + // SOF0-SOF15, minus the four that are not frame headers. + const isFrame = + marker >= 0xc0 && + marker <= 0xcf && + marker !== 0xc4 && + marker !== 0xc8 && + marker !== 0xcc + if (isFrame) return { height: u16(offset + 5), width: u16(offset + 7) } + offset += 2 + u16(offset + 2) + } + } + + return null +} + +/** Whether an image is larger than the long-edge ceiling. */ +export function exceedsEdge(bytes: Uint8Array, edge = MAX_IMAGE_EDGE): boolean { + const size = dimensionsOf(bytes) + return size !== null && Math.max(size.width, size.height) > edge +} + +/** + * Re-encode an oversized image at the long-edge ceiling. + * + * Everything runs in the browser: the bytes are already here, and a round trip + * to a worker to shrink a screenshot would be slower than decoding it in + * place. Injectable so the decision logic above stays testable without a + * canvas. + */ +export type Downscaler = ( + file: File, +) => Promise<{ blob: Blob; mime: string } | null> + +const downscaleInBrowser: Downscaler = async (file) => { + if (typeof createImageBitmap !== 'function') return null + let bitmap: ImageBitmap + try { + bitmap = await createImageBitmap(file) + } catch { + return null + } + try { + const { width, height } = fitWithin(bitmap.width, bitmap.height) + const canvas = document.createElement('canvas') + canvas.width = width + canvas.height = height + const context = canvas.getContext('2d') + if (!context) return null + context.drawImage(bitmap, 0, 0, width, height) + // JPEG for photographs and screenshots alike: a PNG re-encode of a + // photograph is frequently larger than the original, which is the opposite + // of the point. + const blob = await new Promise((resolve) => + canvas.toBlob(resolve, 'image/jpeg', 0.85), + ) + return blob ? { blob, mime: 'image/jpeg' } : null + } finally { + bitmap.close() + } +} + +/** + * Turn every attached image into an image content block. + * + * Attachments without their underlying `File` are skipped silently: a + * conversation reloaded from history keeps the chip, not the bytes. + */ +export async function expandImageAttachments( + attachments: Attachment[], + downscale: Downscaler = downscaleInBrowser, +): Promise { + const candidates = attachments.filter((a) => isImageAttachment(a) && a.file) + if (candidates.length === 0) + return { images: [], blocks: [], read: [], failures: [] } + + const images: AttachmentImageBlock[] = [] + const blocks: string[] = [] + const read: AttachmentReadSummary[] = [] + const failures: AttachmentFailure[] = [] + + const refuse = (attachment: Attachment, reason: string) => { + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + } + + for (const attachment of candidates.slice(0, MAX_IMAGES_PER_SEND)) { + const mime = imageMimeOf(attachment) + const file = attachment.file as File + + if (attachment.size > MAX_SOURCE_IMAGE_BYTES) { + refuse( + attachment, + `${formatBytes(attachment.size)} is too large to send from the composer; point at the file on disk instead`, + ) + continue + } + + const unreadableFormat = !SUPPORTED_IMAGE_MIME.has(mime) + // Both ceilings matter, and neither implies the other: a photograph busts + // the byte limit at a sane resolution, while a flat-coloured screenshot + // eight thousand pixels wide compresses under it and still costs tokens for + // detail no model uses. + const tooLarge = + needsDownscale(file) || + exceedsEdge(new Uint8Array(await file.arrayBuffer())) + // One re-encode covers all three problems: it lands on JPEG at the + // long-edge ceiling, which is a format every model reads and a size every + // model takes. + const converted = + unreadableFormat || tooLarge ? await downscale(file) : null + + if (!converted && (unreadableFormat || tooLarge)) { + refuse( + attachment, + unreadableFormat + ? `${mime} is not a format a model can read, and this browser could not convert it` + : `${formatBytes(attachment.size)} or ${MAX_IMAGE_EDGE}px is over the limit for one image and it could not be resized here`, + ) + continue + } + + const payload: Blob = converted?.blob ?? file + images.push({ + type: 'image', + mime: converted?.mime ?? mime, + data: await fileToBase64(payload), + }) + read.push({ + id: attachment.id, + label: `${attachment.name} · ${formatBytes(payload.size)}${converted ? ' · resized' : ''}`, + }) + } + + reportDropped( + candidates.slice(MAX_IMAGES_PER_SEND), + `only ${MAX_IMAGES_PER_SEND} images are sent per message`, + { blocks, failures }, + ) + + return { images, blocks, read, failures } +} diff --git a/console/web/src/lib/attachments/index.test.ts b/console/web/src/lib/attachments/index.test.ts new file mode 100644 index 000000000..ab10fda72 --- /dev/null +++ b/console/web/src/lib/attachments/index.test.ts @@ -0,0 +1,219 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { Attachment } from '@/types/chat' +import { + classifyAttachment, + expandAttachments, + hasExpandableAttachments, +} from './index' + +vi.mock('@/lib/iii-client', () => ({ + getIiiClient: async () => ({ + trigger: async (functionId: string) => { + if (functionId === 'pdf::classify') { + return { + document_type: 'text_based', + page_count: 2, + pages_needing_ocr: [], + ocr_reasons: [], + elapsed_ms: 3, + } + } + if (functionId === 'pdf::to-markdown') { + return { + body: { + text: 'the pdf text', + chars: 12, + total_chars: 12, + truncated: false, + }, + page_count: 2, + elapsed_ms: 5, + } + } + if (functionId === 'document::to-markdown') { + return { + format: 'docx', + family: 'prose', + detected_from: 'content', + body: { + text: 'the word text', + chars: 13, + total_chars: 13, + truncated: false, + }, + asset_count: 0, + elapsed_ms: 4, + } + } + throw new Error(`unexpected function ${functionId}`) + }, + }), +})) + +function attachment(name: string, type = '', content = 'bytes'): Attachment { + return { + id: name, + name, + size: content.length, + type, + file: new File([content], name, { type }), + } +} + +describe('hasExpandableAttachments', () => { + /* A conversation reloaded from history carries chips, not bytes. Paying for + an expansion pass over them would be work with nothing to show. */ + it('is false when nothing carries its bytes', () => { + const { file, ...chipOnly } = attachment('report.docx') + void file + expect(hasExpandableAttachments([chipOnly])).toBe(false) + expect(hasExpandableAttachments([attachment('report.docx')])).toBe(true) + }) +}) + +describe('classifyAttachment', () => { + /* Every kind an attachment can be, and the one path it takes. Overlaps are + the whole reason this exists. */ + it('gives each overlapping kind exactly one path', () => { + expect( + classifyAttachment(attachment('report.pdf', 'application/pdf')), + ).toBe('pdf') + // A spreadsheet is also plain text; the worker's table is worth more. + expect( + classifyAttachment(attachment('rows.csv', 'application/vnd.ms-excel')), + ).toBe('document') + // An SVG is also an image; no provider decodes one, every model reads the + // markup. + expect(classifyAttachment(attachment('diagram.svg', 'image/svg+xml'))).toBe( + 'text', + ) + expect(classifyAttachment(attachment('shot.png', 'image/png'))).toBe( + 'image', + ) + expect(classifyAttachment(attachment('main.ts', 'video/mp2t'))).toBe('text') + expect( + classifyAttachment(attachment('bundle.zip', 'application/zip')), + ).toBe('unknown') + }) +}) + +describe('expandAttachments', () => { + it('routes each kind down its own path in one pass', async () => { + const result = await expandAttachments([ + attachment('report.pdf', 'application/pdf'), + attachment('quarterly.docx'), + attachment('shot.png', 'image/png'), + attachment('main.ts', '', 'export const x = 1'), + ]) + + expect(result.blocks.join('\n')).toContain('the pdf text') + expect(result.blocks.join('\n')).toContain('the word text') + expect(result.blocks.join('\n')).toContain('export const x = 1') + expect(result.images).toHaveLength(1) + expect(result.images[0].mime).toBe('image/png') + expect(result.failures).toEqual([]) + // One relabelled chip per attachment that was actually read. + expect(result.read).toHaveLength(4) + }) + + /* Live failure this fixes: an SVG was refused as a picture no model can + decode AND inlined as markup that read perfectly well, so the message + carried a "could not read" notice about a file the model had just read. */ + it('sends an SVG as markup only, with no image failure beside it', async () => { + const result = await expandAttachments([ + attachment( + 'diagram.svg', + 'image/svg+xml', + 'flow', + ), + ]) + + expect(result.images).toEqual([]) + expect(result.failures).toEqual([]) + expect(result.blocks).toHaveLength(1) + expect(result.blocks[0]).toContain('flow') + }) + + /* The same overlap on the other side: a CSV is a spreadsheet and plain text, + and used to be converted by the worker and inlined by the browser both. */ + it('sends a CSV through the worker only', async () => { + const result = await expandAttachments([ + attachment('rows.csv', 'application/vnd.ms-excel', 'a,b\n1,2\n'), + ]) + + expect(result.blocks).toHaveLength(1) + expect(result.blocks[0]).toContain('the word text') + expect(result.failures).toEqual([]) + }) + + /* A model with no vision receives an image block and does nothing with it: + the picture disappears downstream and the answer arrives as though nothing + was attached. DeepSeek V4, the cheap default on the rig, is exactly this + case. */ + it('refuses an image when the model cannot see, naming the way out', async () => { + const result = await expandAttachments( + [attachment('shot.png', 'image/png')], + { vision: false, model: 'deepseek-v4-flash' }, + ) + + expect(result.images).toEqual([]) + expect(result.blocks).toHaveLength(1) + expect(result.blocks[0]).toContain('deepseek-v4-flash') + expect(result.blocks[0]).toContain('switch to a model with vision') + expect(result.failures).toHaveLength(1) + }) + + it('sends the image when the model can see', async () => { + const result = await expandAttachments( + [attachment('shot.png', 'image/png')], + { vision: true, model: 'claude-haiku-4-5' }, + ) + + expect(result.images).toHaveLength(1) + expect(result.failures).toEqual([]) + }) + + /* "The catalog did not say" is not "no". An older router, or a model with no + row, must not start eating pictures. */ + it('sends the image when the capability is unknown', async () => { + const result = await expandAttachments([ + attachment('shot.png', 'image/png'), + ]) + + expect(result.images).toHaveLength(1) + expect(result.failures).toEqual([]) + }) + + /* The guard is about pixels only — a spreadsheet still converts on a model + with no vision, because it travels as text. */ + it('leaves documents alone on a model with no vision', async () => { + const result = await expandAttachments([attachment('quarterly.docx')], { + vision: false, + model: 'deepseek-v4-flash', + }) + + expect(result.blocks[0]).toContain('the word text') + expect(result.failures).toEqual([]) + }) + + /* The whole point of the router: a file it cannot read still reaches the + model as a block that says so, rather than vanishing. */ + it('names a file no path can read', async () => { + const result = await expandAttachments([ + attachment('archive.zip', 'application/zip'), + ]) + + expect(result.blocks).toHaveLength(1) + expect(result.blocks[0]).toContain('error=') + expect(result.failures[0].reason).toContain('application/zip') + }) + + it('does no work when nothing carries its bytes', async () => { + const { file, ...chipOnly } = attachment('report.docx') + void file + const result = await expandAttachments([chipOnly]) + + expect(result).toEqual({ blocks: [], images: [], read: [], failures: [] }) + }) +}) diff --git a/console/web/src/lib/attachments/index.ts b/console/web/src/lib/attachments/index.ts new file mode 100644 index 000000000..fda3a2e70 --- /dev/null +++ b/console/web/src/lib/attachments/index.ts @@ -0,0 +1,208 @@ +/** + * One router for everything attached to a message. + * + * The composer accepts anything a person can pick, drag, or paste, and each + * kind reaches a model a different way: a PDF through the `pdf` worker, an + * office document through the `document` worker, an image as a native image + * content block, a text or source file inlined straight from the browser. + * Deciding that in one place is what keeps the send path from growing a branch + * per format. + * + * Nothing here blocks a send. A file that cannot be read becomes a block that + * says so, in the message, where the model can see it — the failure mode this + * whole path exists to prevent is an agent answering as though it had been + * handed nothing. + */ + +import type { Attachment } from '@/types/chat' +import { expandDocumentAttachments, isDocumentAttachment } from './documents' +import { + type ExpandedImages, + expandImageAttachments, + isImageAttachment, +} from './images' +import { expandPdfAttachments, isPdfAttachment } from './pdf' +import { extensionOf, failureBlock, reportDropped } from './shared' +import { expandTextAttachments, isTextAttachment } from './text' + +export { isDocumentAttachment } from './documents' +export { isImageAttachment } from './images' +export { isPdfAttachment } from './pdf' +export type { + AttachmentFailure, + AttachmentImageBlock, + AttachmentReadSummary, +} from './shared' +export { isTextAttachment } from './text' + +import type { + AttachmentFailure, + AttachmentImageBlock, + AttachmentReadSummary, +} from './shared' + +export interface ExpandedAttachments { + /** `` text blocks, appended to the outgoing message. */ + blocks: string[] + /** Native image content blocks, appended after the text. */ + images: AttachmentImageBlock[] + /** New chip labels, keyed by attachment id. */ + read: AttachmentReadSummary[] + /** Everything that could not be read, for the notices above the composer. */ + failures: AttachmentFailure[] +} + +export const EMPTY_EXPANSION: ExpandedAttachments = { + blocks: [], + images: [], + read: [], + failures: [], +} + +/** + * `true` when at least one attachment carries bytes this path can do something + * with. The send path checks this before paying for the expansion, and a + * conversation reloaded from history (chips, no bytes) answers `false`. + */ +export function hasExpandableAttachments(attachments: Attachment[]): boolean { + return attachments.some((a) => a.file) +} + +/** The one path an attachment takes. */ +export type AttachmentKind = 'pdf' | 'document' | 'image' | 'text' | 'unknown' + +/** + * Which path this file takes — exactly one. + * + * The kinds overlap, and letting a file take two paths is not a harmless + * duplicate: an SVG is both `image/svg+xml` and markup, so it was refused as a + * picture no model can decode AND inlined as text that read perfectly well, + * putting a "could not read" notice on a file the model had just read. A CSV is + * both a spreadsheet and plain text, and went through the worker and the + * browser both. + * + * Order is by how much is recovered. A PDF and an office document carry + * structure only their worker can reconstruct. Markup — SVG, HTML, XML — is + * text a model reads directly, and is worth more as characters than as a + * picture it may not be able to decode. A raster image is worth more as pixels + * than as the bytes underneath. Everything else that is text goes in as text. + */ +export function classifyAttachment(attachment: Attachment): AttachmentKind { + if (isPdfAttachment(attachment)) return 'pdf' + if (isDocumentAttachment(attachment)) return 'document' + if (isMarkupAttachment(attachment)) return 'text' + if (isImageAttachment(attachment)) return 'image' + if (isTextAttachment(attachment)) return 'text' + return 'unknown' +} + +/** + * Markup that a browser labels as an image. `image/svg+xml` is the case that + * matters: no provider decodes an SVG as a picture, and every model reads it as + * the markup it is. + */ +function isMarkupAttachment(attachment: Attachment): boolean { + return ( + attachment.type === 'image/svg+xml' || + extensionOf(attachment.name) === 'svg' + ) +} + +export interface ExpandOptions { + /** + * What the model on the other end can do with a picture. `false` refuses + * images with an explanation instead of sending pixels nothing will look at; + * `undefined` means the catalog did not say, and the image goes as before. + */ + vision?: boolean + /** Model id, so the refusal names what to switch away from. */ + model?: string | null +} + +/** + * Expand every attachment on a message. + * + * The four passes run concurrently and the files INSIDE each pass run one at a + * time. That split is the point: two documents queue against the same worker + * rather than opening simultaneous conversions on a machine that is also + * running the model, while a PDF, a spreadsheet and a screenshot — different + * workers, and in the image and text cases no worker at all — have nothing to + * contend over. This sits on the send path, so the message waits for the + * slowest pass rather than the sum of them. + */ +export async function expandAttachments( + attachments: Attachment[], + options: ExpandOptions = {}, +): Promise { + const withBytes = attachments.filter((a) => a.file) + if (withBytes.length === 0) return EMPTY_EXPANSION + + const byKind = new Map() + for (const attachment of withBytes) { + const kind = classifyAttachment(attachment) + byKind.set(kind, [...(byKind.get(kind) ?? []), attachment]) + } + const of = (kind: AttachmentKind) => byKind.get(kind) ?? [] + + const pictures = of('image') + const [pdfs, documents, imagery, texts] = await Promise.all([ + expandPdfAttachments(of('pdf')), + expandDocumentAttachments(of('document')), + // A model with no vision receives an image block and does nothing with it: + // the picture is dropped somewhere downstream and the answer arrives as + // though nothing was attached, the exact silence this whole path exists to + // prevent. Refuse it here, in the message, naming the way out. + options.vision === false + ? refuseImages(pictures, options.model) + : expandImageAttachments(pictures), + expandTextAttachments(of('text')), + ]) + + const blocks: string[] = [ + ...pdfs.blocks, + ...documents.blocks, + ...imagery.blocks, + ...texts.blocks, + ] + const read: AttachmentReadSummary[] = [ + ...pdfs.read, + ...documents.read, + ...imagery.read, + ...texts.read, + ] + const failures: AttachmentFailure[] = [ + ...pdfs.failures, + ...documents.failures, + ...imagery.failures, + ...texts.failures, + ] + const images: AttachmentImageBlock[] = [...imagery.images] + + // Anything left over reached the agent as nothing at all before this router + // existed. Naming it in the message is the whole point: an unreadable + // attachment the model knows about beats a silent one it does not. + for (const attachment of of('unknown')) { + const kind = attachment.type || extensionOf(attachment.name) || 'unknown' + const reason = `${kind} is not a file the console can read into a message` + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + } + + return { blocks, images, read, failures } +} + +/** The image pass, replaced by an explanation, for a model that cannot see. */ +function refuseImages( + pictures: Attachment[], + model: string | null | undefined, +): ExpandedImages { + const refused: ExpandedImages = { + images: [], + blocks: [], + read: [], + failures: [], + } + const reason = `${model ?? 'the selected model'} cannot read images, so this one was not sent — switch to a model with vision` + reportDropped(pictures, reason, refused) + return refused +} diff --git a/console/web/src/lib/pdf-attachments.test.ts b/console/web/src/lib/attachments/pdf.test.ts similarity index 93% rename from console/web/src/lib/pdf-attachments.test.ts rename to console/web/src/lib/attachments/pdf.test.ts index ab948668c..a3d2ae8d2 100644 --- a/console/web/src/lib/pdf-attachments.test.ts +++ b/console/web/src/lib/attachments/pdf.test.ts @@ -6,9 +6,8 @@ import { expandPdfAttachments, isPdfAttachment, MAX_PDFS_PER_SEND, - summaryLabel, TO_MARKDOWN_FUNCTION_ID, -} from './pdf-attachments' +} from './pdf' function pdf(name = 'report.pdf', bytes = 'hello'): Attachment { return { @@ -255,17 +254,10 @@ describe('expandPdfAttachments', () => { const { read } = await expandPdfAttachments([pdf()], fn) expect(read).toHaveLength(1) - expect(read[0]).toMatchObject({ + expect(read[0]).toEqual({ id: 'report.pdf', - pages: 8, - chars: 5932, - elapsedMs: 87, - needsOcr: false, - truncated: false, + label: 'report.pdf · 8 pages · 5,932 chars · 87 ms', }) - expect(summaryLabel('report.pdf', read[0])).toBe( - 'report.pdf · 8 pages · 5,932 chars · 87 ms', - ) }) it('summarizes a scan as unreadable rather than as zero characters', async () => { @@ -279,11 +271,7 @@ describe('expandPdfAttachments', () => { const { read } = await expandPdfAttachments([pdf()], fn) - expect(read[0].needsOcr).toBe(true) - expect(read[0].chars).toBeUndefined() - expect(summaryLabel('scan.pdf', read[0])).toBe( - 'scan.pdf · 3 pages · no readable text · 9 ms', - ) + expect(read[0].label).toBe('report.pdf · 3 pages · no readable text · 9 ms') }) it('marks a truncated extract so the count is not read as the whole document', async () => { @@ -298,8 +286,7 @@ describe('expandPdfAttachments', () => { const { read } = await expandPdfAttachments([pdf()], fn) - expect(read[0].truncated).toBe(true) - expect(summaryLabel('big.pdf', read[0])).toContain('900,000+ chars') + expect(read[0].label).toContain('900,000+ chars') }) it('escapes quotes in a file name rather than breaking the header', async () => { diff --git a/console/web/src/lib/pdf-attachments.ts b/console/web/src/lib/attachments/pdf.ts similarity index 78% rename from console/web/src/lib/pdf-attachments.ts rename to console/web/src/lib/attachments/pdf.ts index 83ac35164..0c54c8ae4 100644 --- a/console/web/src/lib/pdf-attachments.ts +++ b/console/web/src/lib/attachments/pdf.ts @@ -16,8 +16,19 @@ * to the model than an empty one. Failures never block the send. */ -import { getIiiClient } from '@/lib/iii-client' import type { Attachment } from '@/types/chat' +import { + ATTACHED_FILE_PREFIX, + type AttachmentFailure, + type AttachmentReadSummary, + describeWorkerFailure, + escapeAttr, + failureBlock, + fileToBase64, + reportDropped, + type TriggerFn, + triggerOr, +} from './shared' export const CLASSIFY_FUNCTION_ID = 'pdf::classify' export const TO_MARKDOWN_FUNCTION_ID = 'pdf::to-markdown' @@ -39,21 +50,16 @@ export const MAX_MARKDOWN_CHARS = 20_000 */ export const MAX_PDF_BYTES = 64 * 1024 * 1024 -const ATTACHED_FILE_PREFIX = '` block per expanded document, in input order. */ blocks: string[] /** One entry per document actually read, for the message chips. */ - read: PdfReadSummary[] + read: AttachmentReadSummary[] failures: PdfExpansionFailure[] } @@ -104,27 +110,6 @@ interface MarkdownWire { elapsed_ms?: number } -type TriggerFn = ( - functionId: string, - payload: Record, -) => Promise - -/** - * Base64 without building one enormous argument list. - * - * `String.fromCharCode(...bytes)` overflows the call stack somewhere around a - * megabyte, which is a small PDF. Chunking keeps it linear and bounded. - */ -async function fileToBase64(file: File): Promise { - const bytes = new Uint8Array(await file.arrayBuffer()) - const CHUNK = 0x8000 - let binary = '' - for (let i = 0; i < bytes.length; i += CHUNK) { - binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)) - } - return btoa(binary) -} - /** * Read every attached PDF through the `pdf` worker and format the blocks. * @@ -140,15 +125,10 @@ export async function expandPdfAttachments( const pdfs = attachments.filter((a) => isPdfAttachment(a) && a.file) if (pdfs.length === 0) return { blocks: [], read: [], failures: [] } - const call = - trigger ?? - (async (functionId: string, payload: Record) => { - const client = await getIiiClient() - return client.trigger(functionId, payload) - }) + const call = triggerOr(trigger) const blocks: string[] = [] - const read: PdfReadSummary[] = [] + const read: AttachmentReadSummary[] = [] const failures: PdfExpansionFailure[] = [] for (const attachment of pdfs.slice(0, MAX_PDFS_PER_SEND)) { @@ -162,7 +142,10 @@ export async function expandPdfAttachments( try { const outcome = await expandOne(attachment, call) blocks.push(outcome.block) - read.push(outcome.summary) + read.push({ + id: attachment.id, + label: summaryLabel(attachment.name, outcome.summary), + }) } catch (err) { const reason = describeFailure(err) blocks.push(failureBlock(attachment.name, reason)) @@ -170,11 +153,11 @@ export async function expandPdfAttachments( } } - for (const dropped of pdfs.slice(MAX_PDFS_PER_SEND)) { - const reason = `only ${MAX_PDFS_PER_SEND} PDFs are read per message` - blocks.push(failureBlock(dropped.name, reason)) - failures.push({ name: dropped.name, reason }) - } + reportDropped( + pdfs.slice(MAX_PDFS_PER_SEND), + `only ${MAX_PDFS_PER_SEND} PDFs are read per message`, + { blocks, failures }, + ) return { blocks, read, failures } } @@ -275,7 +258,7 @@ async function expandOne( * was read at all, because the expansion happens before the model is called and * so never appears as a function call in the transcript. */ -export function summaryLabel(name: string, summary: PdfReadSummary): string { +function summaryLabel(name: string, summary: PdfReadSummary): string { const parts = [`${summary.pages} page${summary.pages === 1 ? '' : 's'}`] if (summary.needsOcr && summary.chars === undefined) { parts.push('no readable text') @@ -312,30 +295,6 @@ function scannedBlock( ) } -function failureBlock(name: string, reason: string): string { - return `${ATTACHED_FILE_PREFIX}path="${escapeAttr(name)}" error="${escapeAttr(reason)}" />` -} - -/** - * The one failure worth naming precisely: the worker is not installed. Anything - * else surfaces as-is, trimmed. - */ function describeFailure(err: unknown): string { - const message = err instanceof Error ? err.message : String(err) - if (/not registered|NOT_FOUND|function .* not found/i.test(message)) { - return 'the pdf worker is not running — install it with `iii worker add pdf`' - } - return message.length > 160 ? `${message.slice(0, 157)}…` : message -} - -/** - * `>` has to be escaped as well as `&` and `"`. The header is parsed by finding - * the first `>`, so a file name containing one would cut the header short and - * lose every attribute after it. - */ -function escapeAttr(value: string): string { - return value - .replaceAll('&', '&') - .replaceAll('"', '"') - .replaceAll('>', '>') + return describeWorkerFailure(err, 'pdf') } diff --git a/console/web/src/lib/attachments/shared.test.ts b/console/web/src/lib/attachments/shared.test.ts new file mode 100644 index 000000000..d8af7690b --- /dev/null +++ b/console/web/src/lib/attachments/shared.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest' + +import { + describeWorkerFailure, + escapeAttr, + extensionOf, + failureBlock, +} from './shared' + +describe('describeWorkerFailure', () => { + /* The browser SDK rejects with a plain object, so `String(err)` produced the + literal text `[object Object]` — which is exactly what a person saw on the + chip the first time a conversion failed. */ + it('reads the message out of a non-Error rejection', () => { + expect( + describeWorkerFailure({ message: 'malformed document' }, 'document'), + ).toBe('malformed document') + expect( + describeWorkerFailure({ error: 'conversion failed' }, 'document'), + ).toBe('conversion failed') + expect( + describeWorkerFailure( + { error: { message: 'nested detail' } }, + 'document', + ), + ).toBe('nested detail') + }) + + it('never renders an object as [object Object]', () => { + const described = describeWorkerFailure({ status: 500 }, 'document') + expect(described).not.toContain('[object Object]') + expect(described).toContain('500') + }) + + it('names the missing worker and how to install it', () => { + const described = describeWorkerFailure( + { message: 'function document::to-markdown not found' }, + 'document', + ) + expect(described).toContain('iii worker add document') + }) + + it('trims a very long message', () => { + const described = describeWorkerFailure(new Error('x'.repeat(400)), 'pdf') + expect(described.length).toBeLessThanOrEqual(160) + expect(described.endsWith('…')).toBe(true) + }) +}) + +describe('failureBlock', () => { + /* The header is parsed by finding the first `>`, so an unescaped one in a + file name would cut it short and lose every attribute after it. */ + it('escapes the characters that would break the header', () => { + const block = failureBlock('we>ird&"name.docx', 'nope') + expect(block).toContain('>') + expect(block).toContain('&') + expect(block).toContain('"') + // The only unescaped `>` is the one that closes the block. + expect(block.indexOf('>')).toBe(block.length - 1) + }) +}) + +describe('escapeAttr / extensionOf', () => { + it('escapes and extracts as the block writers expect', () => { + expect(escapeAttr('a&b')).toBe('a&b') + expect(extensionOf('Report.FINAL.DocX')).toBe('docx') + expect(extensionOf('Makefile')).toBe('') + expect(extensionOf('.gitignore')).toBe('') + expect(extensionOf('trailing.')).toBe('') + }) +}) diff --git a/console/web/src/lib/attachments/shared.ts b/console/web/src/lib/attachments/shared.ts new file mode 100644 index 000000000..cee392a9e --- /dev/null +++ b/console/web/src/lib/attachments/shared.ts @@ -0,0 +1,173 @@ +/** + * The pieces every attachment kind shares: the envelope a block is written in, + * how bytes get to a worker, and how a failure is phrased. + * + * One envelope for every kind is the point. `` is what + * `#file(...)` mentions already use, so the transcript, the chip renderer and + * the model all see a shape they know, whether the content came from a PDF, a + * spreadsheet, or a text file the browser read directly. + */ + +import { getIiiClient } from '@/lib/iii-client' + +/** Opening of an attachment block; the header ends at the first `>`. */ +export const ATTACHED_FILE_PREFIX = ', +) => Promise + +/** The live trigger, or the caller's stand-in. */ +export function triggerOr(trigger: TriggerFn | undefined): TriggerFn { + return ( + trigger ?? + (async (functionId, payload) => { + const client = await getIiiClient() + return client.trigger(functionId, payload) + }) + ) +} + +/** + * Report the attachments a per-send ceiling cut off. + * + * Every kind has a ceiling and every kind has to say what it dropped: an + * attachment that silently disappears is the failure this whole path exists to + * prevent, and a person who attached six documents deserves to know two of them + * did not go. + */ +export function reportDropped( + dropped: readonly { name: string }[], + reason: string, + into: { blocks: string[]; failures: AttachmentFailure[] }, +): void { + for (const attachment of dropped) { + into.blocks.push(failureBlock(attachment.name, reason)) + into.failures.push({ name: attachment.name, reason }) + } +} + +/** + * Base64 without building one enormous argument list. + * + * `String.fromCharCode(...bytes)` overflows the call stack somewhere around a + * megabyte, which is a small document. Chunking keeps it linear and bounded. + */ +export async function fileToBase64(file: Blob): Promise { + return bytesToBase64(new Uint8Array(await file.arrayBuffer())) +} + +export function bytesToBase64(bytes: Uint8Array): string { + const CHUNK = 0x8000 + let binary = '' + for (let i = 0; i < bytes.length; i += CHUNK) { + binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)) + } + return btoa(binary) +} + +/** + * `>` has to be escaped as well as `&` and `"`. The header is parsed by finding + * the first `>`, so a file name containing one would cut the header short and + * lose every attribute after it. + */ +export function escapeAttr(value: string): string { + return value + .replaceAll('&', '&') + .replaceAll('"', '"') + .replaceAll('>', '>') +} + +/** A self-closing block saying a named file could not be read, and why. */ +export function failureBlock(name: string, reason: string): string { + return `${ATTACHED_FILE_PREFIX}path="${escapeAttr(name)}" error="${escapeAttr(reason)}" />` +} + +/** + * The one failure worth naming precisely: the worker is not installed. Anything + * else surfaces as-is, trimmed. + */ +export function describeWorkerFailure(err: unknown, worker: string): string { + const message = messageOf(err) + if (/not registered|NOT_FOUND|function .* not found/i.test(message)) { + return `the ${worker} worker is not running — install it with \`iii worker add ${worker}\`` + } + return message.length > 160 ? `${message.slice(0, 157)}…` : message +} + +/** + * The readable half of whatever a rejection carried. + * + * The browser SDK rejects with a plain object, not an `Error`, so `String(err)` + * produces the literal text `[object Object]` — which is what the chip and the + * warn notice showed the first time a document failed to convert. Dig for the + * fields a bus error actually carries, and fall back to JSON rather than to a + * sentence that says nothing. + */ +function messageOf(err: unknown): string { + if (err instanceof Error) return err.message + if (typeof err === 'string') return err + if (err && typeof err === 'object') { + const record = err as Record + for (const key of ['message', 'error', 'reason', 'detail']) { + const value = record[key] + if (typeof value === 'string' && value.length > 0) return value + if (value && typeof value === 'object') { + const nested = (value as Record).message + if (typeof nested === 'string' && nested.length > 0) return nested + } + } + if (typeof record.code === 'string') return record.code + try { + return JSON.stringify(err) + } catch { + return 'the worker returned an error with no message' + } + } + return String(err) +} + +/** The file's extension, lowercased, without the dot. Empty when it has none. */ +export function extensionOf(name: string): string { + const dot = name.lastIndexOf('.') + if (dot <= 0 || dot === name.length - 1) return '' + return name.slice(dot + 1).toLowerCase() +} + +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} b` + if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} kb` + return `${(bytes / (1024 * 1024)).toFixed(1)} mb` +} diff --git a/console/web/src/lib/attachments/text.test.ts b/console/web/src/lib/attachments/text.test.ts new file mode 100644 index 000000000..2b90a75e8 --- /dev/null +++ b/console/web/src/lib/attachments/text.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' + +import type { Attachment } from '@/types/chat' +import { + expandTextAttachments, + isTextAttachment, + MAX_TEXT_BYTES, + MAX_TEXT_CHARS, + MAX_TEXT_FILES_PER_SEND, +} from './text' + +function file(name: string, content: string, type = ''): Attachment { + return { + id: name, + name, + size: content.length, + type, + file: new File([content], name, { type }), + } +} + +describe('isTextAttachment', () => { + /* A browser calls a `.ts` file `video/mp2t` — the MPEG transport stream. + Trusting the declared type would inline a video and refuse a TypeScript + file. */ + it('trusts the extension over a wrong MIME type', () => { + expect(isTextAttachment(file('main.ts', 'x', 'video/mp2t'))).toBe(true) + expect(isTextAttachment(file('lib.rs', 'x', ''))).toBe(true) + expect(isTextAttachment(file('notes.md', 'x', ''))).toBe(true) + }) + + it('still accepts anything the browser calls text', () => { + expect(isTextAttachment(file('unknown.conf', 'x', 'text/plain'))).toBe(true) + expect(isTextAttachment(file('data.json', 'x', 'application/json'))).toBe( + true, + ) + }) + + it('leaves documents and images to their own paths', () => { + expect(isTextAttachment(file('report.docx', 'x'))).toBe(false) + expect(isTextAttachment(file('shot.png', 'x', 'image/png'))).toBe(false) + }) +}) + +describe('expandTextAttachments', () => { + it('inlines the file in an attached-file block', async () => { + const result = await expandTextAttachments([ + file('main.ts', 'export const x = 1\n'), + ]) + + expect(result.blocks).toHaveLength(1) + expect(result.blocks[0]).toContain('path="main.ts"') + expect(result.blocks[0]).toContain('export const x = 1') + expect(result.read[0].label).toContain('chars') + }) + + it('truncates a long file and says by how much', async () => { + const long = 'x'.repeat(MAX_TEXT_CHARS + 500) + const result = await expandTextAttachments([file('big.log', long)]) + + expect(result.blocks[0]).toContain('truncated="true"') + expect(result.blocks[0]).toContain(`total-chars="${long.length}"`) + expect(result.read[0].label).toContain('+') + }) + + it('refuses a file over the byte ceiling', async () => { + const attachment = file('huge.log', 'x') + const oversized: Attachment = { ...attachment, size: MAX_TEXT_BYTES + 1 } + const result = await expandTextAttachments([oversized]) + + expect(result.blocks[0]).toContain('error=') + expect(result.failures[0].reason).toContain('limit') + }) + + it('reports the files past the per-send ceiling instead of dropping them', async () => { + const many = Array.from({ length: MAX_TEXT_FILES_PER_SEND + 3 }, (_, i) => + file(`note-${i}.md`, 'hello'), + ) + const result = await expandTextAttachments(many) + + expect(result.read).toHaveLength(MAX_TEXT_FILES_PER_SEND) + expect(result.failures).toHaveLength(3) + }) +}) diff --git a/console/web/src/lib/attachments/text.ts b/console/web/src/lib/attachments/text.ts new file mode 100644 index 000000000..3ca437013 --- /dev/null +++ b/console/web/src/lib/attachments/text.ts @@ -0,0 +1,172 @@ +/** + * Text and source files, inlined directly. + * + * These need no worker: the bytes are already in the browser and they are + * already text. Reading them here rather than routing them through a + * conversion worker keeps a dropped `.ts` file working on a rig where nothing + * but the console is installed. + * + * The extension list exists because a browser's idea of a MIME type is + * unreliable for source code — a `.ts` file arrives as `video/mp2t` (the + * MPEG transport stream), a `.md` as an empty string, a `.rs` as nothing at + * all. Trusting `type` alone would inline a video and refuse a Rust file. + */ + +import type { Attachment } from '@/types/chat' +import { + ATTACHED_FILE_PREFIX, + type AttachmentFailure, + type AttachmentReadSummary, + escapeAttr, + extensionOf, + failureBlock, + formatBytes, + reportDropped, +} from './shared' + +/** Text files inlined per message. */ +export const MAX_TEXT_FILES_PER_SEND = 8 + +/** Characters inlined per file before the block says it stopped short. */ +export const MAX_TEXT_CHARS = 20_000 + +/** Largest text file read from the composer. */ +export const MAX_TEXT_BYTES = 2 * 1024 * 1024 + +/** + * Extensions inlined as text regardless of what the browser calls them. Source + * and configuration files people actually drag into a chat, not an exhaustive + * list — anything missing still inlines when its declared type is `text/*`. + */ +export const TEXT_EXTENSIONS = new Set([ + 'txt', + 'md', + 'markdown', + 'mdx', + 'rst', + 'log', + 'json', + 'jsonl', + 'yaml', + 'yml', + 'toml', + 'ini', + 'env', + 'xml', + 'svg', + 'html', + 'htm', + 'css', + 'scss', + 'sql', + 'sh', + 'bash', + 'zsh', + 'fish', + 'ps1', + 'js', + 'jsx', + 'mjs', + 'cjs', + 'ts', + 'tsx', + 'rs', + 'go', + 'py', + 'rb', + 'java', + 'kt', + 'swift', + 'c', + 'h', + 'cc', + 'cpp', + 'hpp', + 'cs', + 'php', + 'lua', + 'r', + 'dockerfile', + 'gitignore', + 'diff', + 'patch', +]) + +export interface ExpandedText { + blocks: string[] + read: AttachmentReadSummary[] + failures: AttachmentFailure[] +} + +/** Whether an attachment should be inlined as text. */ +export function isTextAttachment(attachment: Attachment): boolean { + if (TEXT_EXTENSIONS.has(extensionOf(attachment.name))) return true + if (attachment.type.startsWith('text/')) return true + return ( + attachment.type === 'application/json' || + attachment.type === 'application/xml' || + attachment.type === 'application/x-yaml' + ) +} + +/** + * Inline every text attachment as an `` block. + * + * Attachments without their underlying `File` are skipped silently: a + * conversation reloaded from history keeps the chip, not the bytes. + */ +export async function expandTextAttachments( + attachments: Attachment[], +): Promise { + const files = attachments.filter((a) => isTextAttachment(a) && a.file) + if (files.length === 0) return { blocks: [], read: [], failures: [] } + + const blocks: string[] = [] + const read: AttachmentReadSummary[] = [] + const failures: AttachmentFailure[] = [] + + for (const attachment of files.slice(0, MAX_TEXT_FILES_PER_SEND)) { + if (attachment.size > MAX_TEXT_BYTES) { + const reason = `${formatBytes(attachment.size)} is over the ${formatBytes(MAX_TEXT_BYTES)} limit for inlining a text file` + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + continue + } + try { + const full = await (attachment.file as File).text() + const truncated = full.length > MAX_TEXT_CHARS + const text = truncated ? full.slice(0, MAX_TEXT_CHARS) : full + + const attrs = [ + `path="${escapeAttr(attachment.name)}"`, + `size="${attachment.size}"`, + ] + if (truncated) { + attrs.push('truncated="true"') + attrs.push(`total-chars="${full.length}"`) + } + const preamble = truncated + ? `This is the first ${MAX_TEXT_CHARS} of ${full.length} characters.\n\n` + : '' + blocks.push( + `${ATTACHED_FILE_PREFIX}${attrs.join(' ')}>\n${preamble}${text}\n`, + ) + read.push({ + id: attachment.id, + label: `${attachment.name} · ${full.length.toLocaleString('en-US')}${truncated ? '+' : ''} chars`, + }) + } catch (err) { + const reason = err instanceof Error ? err.message : String(err) + blocks.push(failureBlock(attachment.name, reason)) + failures.push({ name: attachment.name, reason }) + } + } + + reportDropped( + files.slice(MAX_TEXT_FILES_PER_SEND), + `only ${MAX_TEXT_FILES_PER_SEND} text files are inlined per message`, + { blocks, failures }, + ) + + return { blocks, read, failures } +} diff --git a/console/web/src/lib/backend/harness-send.ts b/console/web/src/lib/backend/harness-send.ts index 91f136cdf..857ef7447 100644 --- a/console/web/src/lib/backend/harness-send.ts +++ b/console/web/src/lib/backend/harness-send.ts @@ -76,14 +76,29 @@ export interface HarnessTextBlock { text: string } +/** + * An image content block on a structured user message — wire-identical to the + * harness's `ContentBlock::Image` (`harness/src/types/content.rs`), which the + * Anthropic and OpenAI providers map onto their own image shapes. `data` is + * base64 without a data-URL prefix. + */ +export interface HarnessImageBlock { + type: 'image' + mime: string + data: string +} + +export type HarnessContentBlock = HarnessTextBlock | HarnessImageBlock + /** * The structured form of `harness::send`'s `message` (MessageInput::Message * with `role: user`). The console uses it when a send carries `#file(...)` - * attachment blocks; plain sends keep the string-sugar form. + * attachment blocks or an attached image; plain sends keep the string-sugar + * form. */ export interface HarnessUserMessage { role: 'user' - content: HarnessTextBlock[] + content: HarnessContentBlock[] timestamp: number } diff --git a/console/web/src/lib/backend/real.ts b/console/web/src/lib/backend/real.ts index 4ee64f26e..0d1b58211 100644 --- a/console/web/src/lib/backend/real.ts +++ b/console/web/src/lib/backend/real.ts @@ -30,6 +30,7 @@ import { loadApprovalGateDefaults } from './approval-gate-config' import { getTurnStatus, type HarnessFunctionPolicy, + type HarnessImageBlock, type HarnessSendRequest, type HarnessThinkingLevel, isTurnActive, @@ -136,14 +137,23 @@ export function buildTurnMetadata( * mention expansions appended. Shared by the send/queue path and the * edit-queued path so an edit rebuilds content exactly as the original did. */ -function buildMessageInput(prompt: string, attachedBlocks: string[]) { - if (attachedBlocks.length === 0) return prompt +function buildMessageInput( + prompt: string, + attachedBlocks: string[], + attachedImages: HarnessImageBlock[] = [], +) { + if (attachedBlocks.length === 0 && attachedImages.length === 0) return prompt return { role: 'user' as const, - content: [prompt, ...attachedBlocks].map((text) => ({ - type: 'text' as const, - text, - })), + content: [ + // Images last: the text says what was asked, and a provider that trims + // content to fit its own window should drop pixels before the question. + ...[prompt, ...attachedBlocks].map((text) => ({ + type: 'text' as const, + text, + })), + ...attachedImages, + ], timestamp: Date.now(), } } @@ -178,7 +188,11 @@ async function buildSendRequest( } } - const message = buildMessageInput(prompt, opts?.attachedBlocks ?? []) + const message = buildMessageInput( + prompt, + opts?.attachedBlocks ?? [], + opts?.attachedImages ?? [], + ) return { session_id: sessionId, @@ -484,13 +498,17 @@ async function realEditQueued( sessionId: string, entryId: string, prompt: string, - opts?: { attachedBlocks?: string[] }, + opts?: { attachedBlocks?: string[]; attachedImages?: HarnessImageBlock[] }, ): Promise { const client = await getIiiClient() await client.trigger('harness::edit_queued', { session_id: sessionId, entry_id: entryId, - message: buildMessageInput(prompt, opts?.attachedBlocks ?? []), + message: buildMessageInput( + prompt, + opts?.attachedBlocks ?? [], + opts?.attachedImages ?? [], + ), }) } diff --git a/console/web/src/lib/backend/types.ts b/console/web/src/lib/backend/types.ts index d47e26092..42deefbaf 100644 --- a/console/web/src/lib/backend/types.ts +++ b/console/web/src/lib/backend/types.ts @@ -1,4 +1,5 @@ import type { Mode, ModelId } from '@/types/chat' +import type { HarnessImageBlock } from './harness-send' import type { SessionTriggerInfo } from './triggers' /** @@ -142,6 +143,14 @@ export interface ChatStreamOptions { * backends ignore this. */ attachedBlocks?: string[] + /** + * Image content blocks appended after the text on the outgoing user message + * — an attached, dropped or pasted picture, sent as a picture rather than as + * prose about one. The real backend forwards them as + * `ContentBlock::Image { mime, data }`, which the Anthropic and OpenAI + * providers map onto their own image shapes. Mock backends ignore this. + */ + attachedImages?: HarnessImageBlock[] /** mean delay between assistant tokens, in ms */ meanDelayMs?: number /** @@ -300,7 +309,10 @@ export interface ChatBackend { sessionId: string, entryId: string, prompt: string, - opts?: { attachedBlocks?: string[] }, + opts?: { + attachedBlocks?: string[] + attachedImages?: HarnessImageBlock[] + }, ): Promise /** * Subscribe to `harness::message-queued` for a session: fires when any diff --git a/console/web/src/lib/file-mentions.ts b/console/web/src/lib/file-mentions.ts index f9676c386..ccf9f6806 100644 --- a/console/web/src/lib/file-mentions.ts +++ b/console/web/src/lib/file-mentions.ts @@ -10,7 +10,13 @@ * `failures` entry the caller surfaces as a chat notice. */ -import { getIiiClient } from '@/lib/iii-client' +import { + ATTACHED_FILE_PREFIX, + escapeAttr, + failureBlock, + type TriggerFn, + triggerOr, +} from '@/lib/attachments/shared' export const READ_FILE_FUNCTION_ID = 'coder::read-file' @@ -20,8 +26,6 @@ const FILE_MENTION_RE = /#file\(([^)]+)\)/g /** Max unique mentions expanded per send; extras are ignored. */ export const MAX_MENTIONS_PER_SEND = 20 -const ATTACHED_FILE_PREFIX = ', -) => Promise - /** * Read every mentioned file in one jail-validated batch call and format the * attachment blocks. Batch results come back in request order (the wire @@ -95,12 +94,7 @@ export async function expandFileMentions( return { blocks: [], attachments: [], failures: [] } } - const call = - trigger ?? - (async (functionId: string, payload: Record) => { - const client = await getIiiClient() - return client.trigger(functionId, payload) - }) + const call = triggerOr(trigger) let results: ReadEntryResultWire[] try { @@ -152,10 +146,6 @@ function contentBlock(path: string, entry: ReadEntryResultWire): string { return `${ATTACHED_FILE_PREFIX}${attrs.join(' ')}>\n${entry.content}\n` } -function failureBlock(path: string, reason: string): string { - return `${ATTACHED_FILE_PREFIX}path="${escapeAttr(path)}" error="${escapeAttr(reason)}" />` -} - function shortReason(message: string | undefined | null): string | undefined { if (!message) return undefined return message.length > 120 ? `${message.slice(0, 117)}…` : message @@ -196,10 +186,9 @@ export function parseAttachedFileHeader( } } -function escapeAttr(value: string): string { - return value.replaceAll('&', '&').replaceAll('"', '"') -} - function unescapeAttr(value: string): string { - return value.replaceAll('"', '"').replaceAll('&', '&') + return value + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll('&', '&') } diff --git a/console/web/src/lib/models-catalog.test.ts b/console/web/src/lib/models-catalog.test.ts index 6870f893e..96b3151aa 100644 --- a/console/web/src/lib/models-catalog.test.ts +++ b/console/web/src/lib/models-catalog.test.ts @@ -29,4 +29,37 @@ describe('catalogRowsToModelOptions', () => { ], }) }) + + /* The send path refuses to hand a picture to a model that cannot see one, so + this flag has to survive the catalog. It stays TRI-state: a router that + says nothing must not read as "no", or every model on an older catalog + would start rejecting images. */ + it('carries vision support through, including "not stated"', () => { + const rows: CatalogModelRow[] = [ + { + id: 'deepseek-v4-flash', + provider: 'deepseek', + display_name: 'DeepSeek V4 Flash', + supports_vision: false, + }, + { + id: 'claude-haiku-4-5', + provider: 'anthropic', + display_name: 'Claude Haiku 4.5', + supports_vision: true, + }, + { + id: 'mystery-1', + provider: 'somewhere', + display_name: 'Mystery 1', + }, + ] + + const byId = new Map( + catalogRowsToModelOptions(rows).map((o) => [o.id, o.supportsVision]), + ) + expect(byId.get('deepseek::deepseek-v4-flash')).toBe(false) + expect(byId.get('anthropic::claude-haiku-4-5')).toBe(true) + expect(byId.get('somewhere::mystery-1')).toBeUndefined() + }) }) diff --git a/console/web/src/lib/models-catalog.ts b/console/web/src/lib/models-catalog.ts index 547701f2e..c95f6efdf 100644 --- a/console/web/src/lib/models-catalog.ts +++ b/console/web/src/lib/models-catalog.ts @@ -9,6 +9,8 @@ export interface CatalogModelRow { display_name: string context_window?: number supports_thinking?: boolean + /** Absent when the router says nothing about it — see `ModelOption`. */ + supports_vision?: boolean reasoning_efforts?: ReasoningEffortOption[] } @@ -55,6 +57,8 @@ export async function fetchModelsCatalog(): Promise { : undefined const supports_thinking = typeof o.supports_thinking === 'boolean' ? o.supports_thinking : undefined + const supports_vision = + typeof o.supports_vision === 'boolean' ? o.supports_vision : undefined const reasoning_efforts = parseReasoningEfforts(o.reasoning_efforts) if (!id || !provider) continue out.push({ @@ -63,6 +67,7 @@ export async function fetchModelsCatalog(): Promise { display_name, context_window, supports_thinking, + supports_vision, reasoning_efforts, }) } @@ -80,6 +85,10 @@ export function catalogRowsToModelOptions( label: m.display_name.toLowerCase(), contextWindow: m.context_window, supportsThinking: m.supports_thinking === true, + // Kept tri-state, unlike `supportsThinking`: "the router did not say" has + // to stay distinguishable from "no", or every model on an older catalog + // would refuse images. + supportsVision: m.supports_vision, reasoningEfforts: m.reasoning_efforts, })) } diff --git a/console/web/src/lib/sessions/entry-mapper.test.ts b/console/web/src/lib/sessions/entry-mapper.test.ts index 7e73f3cc7..1d9372b4f 100644 --- a/console/web/src/lib/sessions/entry-mapper.test.ts +++ b/console/web/src/lib/sessions/entry-mapper.test.ts @@ -111,6 +111,36 @@ describe('entrySegments', () => { expect((msg as { content: string }).content).not.toContain('fn main') }) + /* A pasted screenshot IS the message's content for a vision model. Dropping + the block on the way in would leave a reloaded conversation showing the + question with no sign a picture went with it. */ + it('turns image blocks into chips that keep their thumbnail', () => { + const item: TranscriptItem = { + entry_id: 'msg-3-user-0', + message: { + role: 'user', + content: [ + { type: 'text', text: 'what is wrong with this screen?' }, + { type: 'image', mime: 'image/png', data: 'AAAA' }, + ], + timestamp: 1, + }, + } + const [msg] = entrySegments(item) + expect(msg).toMatchObject({ + role: 'user', + content: 'what is wrong with this screen?', + attachments: [ + { + id: 'image-1', + name: 'image 1', + type: 'image/png', + dataUrl: 'data:image/png;base64,AAAA', + }, + ], + }) + }) + it('marks only trusted notification user entries', () => { expect( entrySegments(userItem('e-1', 'normal', { notification: false }))[0], diff --git a/console/web/src/lib/sessions/entry-mapper.ts b/console/web/src/lib/sessions/entry-mapper.ts index 9241e4716..ebc5612e7 100644 --- a/console/web/src/lib/sessions/entry-mapper.ts +++ b/console/web/src/lib/sessions/entry-mapper.ts @@ -296,10 +296,14 @@ function textOf(blocks: ContentBlock[]): string { /** * Split a user message's blocks into visible text and attachment chips. - * `` blocks are console-authored `#file(...)` mention - * expansions — rendering their full content in the user bubble would dump - * whole files into the chat, so they collapse to chips instead (failure + * `` blocks are console-authored `#file(...)` mention and + * document expansions — rendering their full content in the user bubble would + * dump whole files into the chat, so they collapse to chips instead (failure * placeholders keep the error visible in the chip name). + * + * An image block is the picture itself, sent to a vision model. It becomes a + * chip carrying its own thumbnail: without this a conversation reloaded from + * history shows the question and no sign that a screenshot went with it. */ function splitUserContent(blocks: ContentBlock[]): { text: string @@ -307,7 +311,22 @@ function splitUserContent(blocks: ContentBlock[]): { } { let text = '' const attachments: Attachment[] = [] + let imageIndex = 0 for (const block of blocks) { + if (block.type === 'image') { + imageIndex += 1 + const mime = block.mime || 'image/png' + attachments.push({ + id: `image-${imageIndex}`, + name: `image ${imageIndex}`, + // Base64 inflates by a third; the original byte count is what a + // person recognises, so report that rather than the encoded length. + size: Math.floor((block.data?.length ?? 0) * 0.75), + type: mime, + dataUrl: block.data ? `data:${mime};base64,${block.data}` : undefined, + }) + continue + } if (block.type !== 'text') continue const header = parseAttachedFileHeader(block.text) if (header) { diff --git a/console/web/src/types/chat.ts b/console/web/src/types/chat.ts index 2890729de..cb446cbcb 100644 --- a/console/web/src/types/chat.ts +++ b/console/web/src/types/chat.ts @@ -12,6 +12,12 @@ export interface ModelOption { label: string contextWindow?: number supportsThinking?: boolean + /** + * Whether the model reads images. `undefined` means the router did not say — + * an older catalog, or a model it has no row for — and callers treat that as + * "assume it can" rather than refusing to send a picture on missing metadata. + */ + supportsVision?: boolean reasoningEfforts?: ReasoningEffortOption[] } diff --git a/document/Cargo.lock b/document/Cargo.lock new file mode 100644 index 000000000..ee22ae1ef --- /dev/null +++ b/document/Cargo.lock @@ -0,0 +1,2991 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anydoc" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93a8b0dbecf79be9329111093b493d256db2a573ed3f1cc827c96bdbbe32a936" +dependencies = [ + "calamine", + "cfb", + "csv", + "encoding_rs", + "flate2", + "log", + "pdf-inspector", + "quick-xml", + "zip", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "chrono", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a347dcabdae9c31b0825fd6a8bed285ec9c2acb89c47827126d52fa4f59cece3" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "document" +version = "0.1.0" +dependencies = [ + "anydoc", + "anyhow", + "base64", + "clap", + "iii-sdk", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fast-float2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "iii-helpers" +version = "0.21.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bdc7bbc3abfde934a62cdc5d3045adf52914dfc1ed6c20f8af691fc561dc55" +dependencies = [ + "futures-util", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "reqwest", + "schemars", + "serde", + "serde_json", + "sysinfo", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "iii-sdk" +version = "0.21.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07dd060fddcc9153b0dd07c038a14cf172ce15ce1d4edb98155563ed55b2caba" +dependencies = [ + "async-trait", + "futures-util", + "hostname", + "iii-helpers", + "reqwest", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", +] + +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap", + "itoa", + "jiff", + "log", + "md-5", + "nom", + "rand 0.10.2", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "rand 0.9.5", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "pdf-inspector" +version = "1.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e024ae242c514e2adf6aee186678e0eabdc2e5ecfbb2159186881b4498593cb" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.119", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/document/Cargo.toml b/document/Cargo.toml new file mode 100644 index 000000000..1035ec717 --- /dev/null +++ b/document/Cargo.toml @@ -0,0 +1,36 @@ +[workspace] + +[package] +name = "document" +version = "0.1.0" +edition = "2021" +description = "Document worker for iii — convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV and PDF to markdown on the machine, detect the format from the bytes, and pull out embedded images (document::* functions)" +license = "Apache-2.0" +repository = "https://github.com/iii-hq/workers" +publish = false + +[lib] +name = "document" +path = "src/lib.rs" + +[[bin]] +name = "document" +path = "src/main.rs" + +[dependencies] +iii-sdk = "=0.21.6" +# The converter. Pure Rust, no system libraries, no subprocess, no network. +anydoc = "=0.1.9" +base64 = "0.22" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" +anyhow = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } +clap = { version = "4", features = ["derive", "env"] } +schemars = "0.8" + +[dev-dependencies] +tempfile = "3" diff --git a/document/README.md b/document/README.md new file mode 100644 index 000000000..b0a256780 --- /dev/null +++ b/document/README.md @@ -0,0 +1,251 @@ +# document + +Read office documents on the machine, with no conversion service and no API +key. This worker takes a Word, PowerPoint, Excel, OpenDocument, RTF, EPUB or CSV +file and returns markdown that keeps its headings, lists, tables and notes, in +single-digit milliseconds for a typical document. It identifies a file from its +bytes rather than trusting its name, so a mislabelled attachment still converts. +And it hands back the images markdown cannot carry, which is what a deck of +diagrams actually holds. Nothing is uploaded, and a long document is capped +rather than dumped, so a report does not swallow the context an agent needed for +the answer. + +## Install + +```bash +iii worker add document +``` + +Reading a scanned document also needs something to turn its pages into pixels +and something to read them, neither of which ships here: + +```bash +iii worker add browser +``` + +With [browser](https://github.com/iii-hq/workers/tree/main/browser) installed and a vision model configured through +[llm-router](https://github.com/iii-hq/workers/tree/main/llm-router), `document::ocr` transcribes scans. Every other +function works without both. + +## Quickstart + +```rust +use iii_sdk::{register_worker, InitOptions}; +use iii_sdk::protocol::TriggerRequest; +use serde_json::json; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let iii = register_worker("ws://localhost:49134", InitOptions::default()); + + let markdown = iii.trigger(TriggerRequest { + function_id: "document::to-markdown".into(), + payload: json!({ "path": "/tmp/quarterly.docx" }), + action: None, + timeout_ms: Some(60_000), + }).await?; + // { "format": "docx", "family": "prose", "detected_from": "content", + // "body": { "text": "# Quarterly Notes\n…", "chars": 5693, + // "total_chars": 5693, "truncated": false }, + // "asset_count": 0, "elapsed_ms": 4, … } + + println!("{markdown:#?}"); + Ok(()) +} +``` + +A document with no path goes in as `bytes_base64` instead — the shape a composer +attachment takes. Add `file_name` with it: a CSV carries no signature of its +own, and without a name it cannot be recognised. + +## Formats + +| Format | Extensions | +|---|---| +| Word | `.doc`, `.docx`, `.docm` | +| PowerPoint | `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm` | +| Excel | `.xls`, `.xlsx`, `.xlsm`, `.xlsb` | +| OpenDocument | `.odt`, `.ods`, `.odp` | +| Rich Text | `.rtf` | +| EPUB | `.epub` | +| CSV | `.csv` | +| PDF | `.pdf` (text-based; see below) | + +Container variants collapse onto one name: `.docm` is `docx`, `.xlsb` is +`excel`. A caller matches on the format, never on the extension it happened to +send. + +### PDFs + +A text-based PDF converts here, which makes this worker a complete answer for a +mixed pile of attachments on its own. When the [`pdf`](https://github.com/iii-hq/workers/tree/main/pdf) worker is +installed it is the better route for them: it classifies scanned versus +text-based and names the individual pages that need OCR, where this worker can +only convert or fail. + +## Detect before you convert + +`document::detect` reads the signature in the first bytes of a file and answers +in microseconds. It exists for the case where something arrives and nobody knows +what it is. + +```json +{ + "format": "pptx", + "family": "presentation", + "detected_from": "content", + "convertible": true, + "has_assets": true, + "size_bytes": 184320, + "source": "roadmap.pptx", + "elapsed_ms": 0 +} +``` + +`detected_from` is the field worth reading. `content` means the bytes named the +format, which is the strong answer. `extension` means they did not, and only the +file name suggested it — expected for a CSV, and a reason for suspicion on +anything else. A `format` of `null` is an answer too: this is not a document +this worker reads, not a document that is broken. + +## The images markdown drops + +Markdown renders an embedded image as its alt text. For prose that is right. For +a deck built out of diagrams it throws away the content and leaves a page of +titles, which reads as a document that had little to say. + +`document::to-markdown` reports `asset_count` so that case is visible, and +`document::extract-assets` returns the bytes: + +```json +{ + "format": "pptx", + "assets": [ + { + "index": 0, + "media_type": "image/png", + "origin_part": "ppt/media/image1.png", + "size_bytes": 48211, + "bytes_base64": "iVBORw0KGgo…" + } + ], + "total_count": 1, + "truncated": false +} +``` + +Three ceilings apply, and all of them report what they dropped rather than +trimming silently. `max_assets` bounds how many come back. `max_asset_bytes` +bounds one payload, and `max_assets_total_bytes` bounds the response as a whole, +because two dozen assets each just under the per-asset limit still add up to a +quarter of a gigabyte once base64 inflates them. An asset left out either way is +still listed with its type and size, with `omitted` saying which ceiling it hit +(`too_large` or `budget_spent`), so a caller can ask for it on its own. +`include_bytes: false` inventories a document without moving anything. + +## Reading a scan + +A scanned page holds no text to extract: the characters exist only in the +pixels. `document::ocr` renders those pages and reads them with a vision model. + +```json +{ + "via": "pdf-render", + "body": { "text": "INVOICE 4471\nDue 30 June…", "chars": 812, "truncated": false }, + "pages": [{ "page": 1, "text": "INVOICE 4471…", "chars": 812, "cached": false }], + "pages_transcribed": 1, + "pages_cached": 0, + "model": "claude-haiku-4-5" +} +``` + +Three inputs, one answer. An image goes straight to the model. A PDF is +rendered a page at a time by the [`browser`](https://github.com/iii-hq/workers/tree/main/browser) worker, which is the +only thing that turns a page into pixels. An office document whose text came +back empty has its embedded images pulled out and read the same way. + +Both of those dependencies are soft. Neither is declared in +`iii.worker.yaml`, every other function works without them, and a call that +needs one it cannot reach says which to install. Someone who installed this +worker to read a `.docx` never pays for Chromium. + +This is the one function here that costs money, so nothing runs it implicitly. +`pdf::classify` reports which pages are scans, and passing that list is the +difference between transcribing one page of a report and all four hundred: + +```json +{ "path": "/tmp/report.pdf", "pages": [1], "model": "claude-haiku-4-5" } +``` + +The model is checked for vision support before anything is rendered, because a +model that cannot see fails on the first page after the render has been paid +for. + +Page transcriptions cache in the `state` worker, keyed by the rendered PIXELS +and the model that read them. Keying on the image rather than the source +document is what makes the cache self-correcting: a page that rendered badly +hashes differently once the render is fixed, so bad entries fall out instead of +being served forever. A hit still re-renders — that is a second of local +Chromium — and skips the model call, which is the part that costs money. The +images themselves are never stored, and exist only in flight between the browser +and the model. + +For scale: one rendered page of a text PDF measured about 1,400 input tokens on +`claude-haiku-4-5`, or roughly $0.0016 a page. + +Rendering a PDF needs the file on disk (`path`, not `bytes_base64`) and the +`browser` worker allowed to open it: its Behavior settings carry an allowed +URL schemes list that ships as `http, https`, and a local PDF needs `file` +added. It hot-applies on save. That list is deliberately narrow — the browser +does not check a path against the session's filesystem scope the way this +worker does, so widening it widens what any caller can read. + +## Response caps + +Every text-bearing response is capped and says so. `truncated: true` with a +`total_chars` far above `chars` means you are holding a fragment. `max_chars: 0` +takes the whole document, and belongs in a pipeline moving a document to +storage rather than in a call whose result lands in a conversation. + +## Configuration + +Configuration lives in the `configuration` worker under the id `document` and +every field hot-reloads. Nothing here needs a restart. + +```yaml +max_input_bytes: 67108864 # largest document accepted, before parsing +max_chars: 40000 # default cap on returned markdown +preview_chars: 600 # leading characters shown alongside a capped body +max_assets: 24 # assets returned in one response +max_asset_bytes: 8388608 # largest single asset returned with its bytes +max_assets_total_bytes: 33554432 # total asset payload one response may carry +ocr_model: # vision model document::ocr reads with; unset = every call chooses +max_ocr_pages: 20 # pages one document::ocr call transcribes +ocr_timeout_ms: 120000 # budget for one render or one model read +ocr_render_settle_ms: 2000 # let a rendered page paint before capturing it +ocr_cache: true # cache page transcriptions in the state worker +``` + +A per-call `max_assets` narrows this ceiling and cannot raise it: the limit +bounds one response, and a caller asking for a thousand images is the case it +exists for. + +Defaults live in [`src/config.rs`](src/config.rs). + +## Called on demand + +This worker registers no harness hook and injects nothing into any prompt. A +conversation that never touches a document never pays for it, and there is no +per-turn cost to having it installed. An agent finds it the ordinary way, +through the function registry and [`skills/SKILL.md`](skills/SKILL.md). + +## What this worker does not do + +It does not run OCR by itself. `document::ocr` renders and asks a model, which +means a scan costs money per page and needs a vision model configured. Nothing +transcribes implicitly. + +It does not write documents. Conversion is one way, into markdown. + +It cannot open an encrypted document. There is no password parameter, because +there is nothing behind it that could decrypt one. diff --git a/document/build.rs b/document/build.rs new file mode 100644 index 000000000..9aaeeedcd --- /dev/null +++ b/document/build.rs @@ -0,0 +1,12 @@ +//! Build script for the `document` worker. +//! +//! One job: forward the build-time target triple to the binary as +//! `env!("TARGET")`, which `manifest.rs` reports as the registry's +//! `supported_targets` field. + +fn main() { + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); +} diff --git a/document/iii.worker.yaml b/document/iii.worker.yaml new file mode 100644 index 000000000..3b3fe8115 --- /dev/null +++ b/document/iii.worker.yaml @@ -0,0 +1,11 @@ +iii: v1 +name: document +language: rust +deploy: binary +manifest: Cargo.toml +license: Apache-2.0 +bin: document +tags: [document, markdown, docx, pptx, xlsx, epub, csv, attachments, text-extraction, ocr] +description: Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV and PDF documents to markdown on this machine, detect the format from the bytes, pull out the images embedded in them, and transcribe a scan by rendering its pages and reading them with a vision model. +dependencies: + configuration: "^0.21.6" diff --git a/document/skills/SKILL.md b/document/skills/SKILL.md new file mode 100644 index 000000000..77a2fa438 --- /dev/null +++ b/document/skills/SKILL.md @@ -0,0 +1,85 @@ +--- +name: document +description: >- + Read Word, PowerPoint, Excel, OpenDocument, RTF, EPUB and CSV files locally + with no API key — detect the format from the bytes, convert to markdown with + headings, lists and tables intact, and pull out the images markdown drops. +--- + +# document + +The document worker converts office documents on the machine. A `.docx` or a +`.pptx` is a ZIP of XML: reading one with a file-reading function returns +compressed noise and spends the context on it, so every office document goes +through `document::*` instead. Conversion is local, needs no credential, and +sends nothing anywhere. + +One serializer sits behind every format, so a `.doc` from 2003 and a `.pptx` +from yesterday come out with the same heading, table and list conventions. That +sameness is the point: a conversation handling a mixed bag of attachments reads +one shape, not fourteen. + +The one thing markdown cannot carry is the pictures. An embedded image renders +as its alt text, which is right for prose and wrong for a deck of diagrams — a +deck whose content is images converts to a page of titles and reads as an empty +document. `document::to-markdown` reports how many images it dropped, and +`document::extract-assets` returns their bytes for a model that can see them. + +This worker is called on demand. It registers no harness hook and injects +nothing into any prompt, so a conversation that never touches a document never +pays for it. Reach for it when one appears. + +## When to Use + +- A conversation names or hands over a `.docx`, `.doc`, `.pptx`, `.ppt`, + `.xlsx`, `.xls`, `.odt`, `.ods`, `.odp`, `.rtf`, `.epub` or `.csv`: call + `document::to-markdown`. Never read one with a file-reading function. +- A file whose type is unclear, or a batch to route: `document::detect` first. + It reads the signature in the first bytes and answers in microseconds. +- The markdown came back thin and `asset_count` is above zero: the content is + pictures. Call `document::extract-assets` and hand the images to a model that + can see them. +- A PDF: prefer `pdf::classify` and `pdf::to-markdown` when the `pdf` worker is + installed — it reports which pages are scans and need OCR. This worker + converts text-based PDFs too, as a fallback. +- A document that came back with no text, or one classified as a scan: say so + and offer `document::ocr` rather than running it unasked. It costs money per + page. When you do run it, pass the `pages` that `pdf::classify` named. + +## Boundaries + +- `document::ocr` is the only function here that spends money, and the only one + that needs other workers: `browser` to render a PDF's pages, and a vision + model through llm-router. Both are optional installs; a call that needs one + it cannot reach says which. Rendering a PDF needs `path`, not + `bytes_base64`. +- Nothing here writes documents. Conversion is one-way, to markdown. +- Responses are capped. `truncated: true` with a much larger `total_chars` + means you hold a fragment and must not answer from it. `max_chars: 0` lifts + the cap and belongs in a pipeline moving a document to storage, not in a call + whose result lands in the conversation. +- `document::extract-assets` is capped twice: how many assets come back, and + how large one may be before its bytes are left out. Anything left out is + still listed with its media type and size — an empty list means the document + genuinely holds nothing. +- A CSV carries no signature, so it is recognised only by its file name. Inline + bytes need `file_name` for it; every other format is read from the content. +- `detected_from: "extension"` on anything other than a CSV means the content + matched nothing known and only the name suggested the format. Treat the + result with more suspicion than a `content` detection. +- An encrypted document cannot be opened here at all. There is no password + parameter; ask for an unlocked copy. + +## Functions + +- `document::detect` — what this file is, from its bytes: the format, the + family (prose, spreadsheet, presentation, book, PDF), how it was recognised, + and whether it can be converted. Microseconds, and no conversion. +- `document::to-markdown` — the document as markdown, with headings, lists, + links, tables, footnotes and speaker notes preserved. Reports the count of + embedded images it could not carry. +- `document::extract-assets` — the embedded images and objects as base64, + filtered by media type, capped per response and per asset. +- `document::ocr` — transcribe a document that holds no readable text: a + scanned PDF, a photographed page, a deck built out of pictures. Renders the + pages and reads them with a vision model. diff --git a/document/src/bus.rs b/document/src/bus.rs new file mode 100644 index 000000000..5c8cbc252 --- /dev/null +++ b/document/src/bus.rs @@ -0,0 +1,218 @@ +//! Calling other workers, and being able to test that we did. +//! +//! Every function in this worker except `document::ocr` is pure CPU work over +//! a buffer. OCR is the exception: it needs pixels it cannot produce and a +//! model it does not host, so it talks to `browser` and `llm-router` over the +//! bus. +//! +//! Those calls go through this trait rather than an `IIIClient` directly, for +//! two reasons. A test can drive the whole handler — render, transcribe, cache +//! — against recorded responses with no engine, no Chromium and no model bill. +//! And the dependency stays SOFT: nothing here is declared in +//! `iii.worker.yaml`, so a worker that is not installed surfaces as a failed +//! call this module turns into an instruction, not as a boot-time refusal that +//! would cost a `.docx` reader a browser install. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use iii_sdk::protocol::TriggerRequest; +use iii_sdk::IIIClient; +use serde_json::Value; + +pub type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// One bus call: a function id, a payload, a JSON answer or a message. +pub trait Bus: Send + Sync { + fn trigger<'a>( + &'a self, + function_id: &'a str, + payload: Value, + timeout_ms: u64, + ) -> BoxFuture<'a, Result>; +} + +/// The live bus. +pub struct EngineBus { + iii: Arc, +} + +impl EngineBus { + pub fn new(iii: Arc) -> Self { + Self { iii } + } +} + +impl Bus for EngineBus { + fn trigger<'a>( + &'a self, + function_id: &'a str, + payload: Value, + timeout_ms: u64, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload, + action: None, + timeout_ms: Some(timeout_ms), + }) + .await + .map_err(|e| e.to_string()) + }) + } +} + +/// The worker a failed call was trying to reach, for the message a caller acts +/// on. +/// +/// "remote error (NOT_FOUND)" tells someone nothing. "the browser worker is not +/// installed" tells them the one thing they can do about it, which matters more +/// here than anywhere else in this worker: OCR is the only surface whose +/// dependencies are not shipped with it. +pub fn describe_bus_failure(function_id: &str, err: &str) -> String { + let worker = function_id.split("::").next().unwrap_or(function_id); + let missing = err.to_ascii_uppercase().contains("NOT_FOUND") + || err.contains("not registered") + || err.contains("not found"); + if !missing { + return format!("{function_id} failed: {err}"); + } + match worker { + "browser" => "reading a scanned PDF needs the browser worker to render its pages; \ + install it with `iii worker add browser`" + .to_string(), + "router" => "transcribing needs a model through llm-router; install it with \ + `iii worker add llm-router` and configure a provider" + .to_string(), + "state" => format!("{function_id} is unavailable: {err}"), + _ => format!("{function_id} is not available: {err}"), + } +} + +#[cfg(test)] +pub mod test_bus { + //! A recorded bus: each function id answers with a queued value or an + //! error, and every call is logged so a test can assert the order things + //! happened in — that a page was rendered before it was transcribed, or + //! that a cached page was never rendered at all. + + use std::collections::HashMap; + use std::sync::Mutex; + + use super::*; + + #[derive(Default)] + pub struct RecordedBus { + responses: Mutex>>>, + pub calls: Mutex>, + } + + impl RecordedBus { + pub fn new() -> Self { + Self::default() + } + + /// Queue one answer for `function_id`. Repeated pushes answer repeated + /// calls in order; the last answer repeats once the queue is empty. + pub fn on(self, function_id: &str, value: Value) -> Self { + self.responses + .lock() + .expect("lock") + .entry(function_id.to_string()) + .or_default() + .push(Ok(value)); + self + } + + pub fn failing(self, function_id: &str, error: &str) -> Self { + self.responses + .lock() + .expect("lock") + .entry(function_id.to_string()) + .or_default() + .push(Err(error.to_string())); + self + } + + pub fn called(&self) -> Vec { + self.calls + .lock() + .expect("lock") + .iter() + .map(|(id, _)| id.clone()) + .collect() + } + + pub fn payloads(&self, function_id: &str) -> Vec { + self.calls + .lock() + .expect("lock") + .iter() + .filter(|(id, _)| id == function_id) + .map(|(_, payload)| payload.clone()) + .collect() + } + } + + impl Bus for RecordedBus { + fn trigger<'a>( + &'a self, + function_id: &'a str, + payload: Value, + _timeout_ms: u64, + ) -> BoxFuture<'a, Result> { + self.calls + .lock() + .expect("lock") + .push((function_id.to_string(), payload)); + let mut responses = self.responses.lock().expect("lock"); + let queued = responses.get_mut(function_id); + let answer = match queued { + Some(queue) if queue.len() > 1 => queue.remove(0), + Some(queue) if queue.len() == 1 => queue[0].clone(), + _ => Err(format!( + "remote error (NOT_FOUND): {function_id} not registered" + )), + }; + Box::pin(async move { answer }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The point of the whole module: a missing worker has to arrive as an + /// instruction, because OCR is the one surface here whose dependencies are + /// not shipped with the binary. + #[test] + fn a_missing_worker_becomes_something_to_do() { + let browser = describe_bus_failure( + "browser::screenshot", + "remote error (NOT_FOUND): browser::screenshot not registered", + ); + assert!(browser.contains("iii worker add browser"), "{browser}"); + + let router = describe_bus_failure( + "router::complete", + "remote error (NOT_FOUND): router::complete not registered", + ); + assert!(router.contains("llm-router"), "{router}"); + } + + /// A real failure from a worker that IS there passes through: the caller + /// needs the reason, not advice to install something already installed. + #[test] + fn a_live_worker_failure_is_reported_as_it_came() { + let described = describe_bus_failure("browser::navigate", "scheme `file` is not allowed"); + assert!( + described.contains("scheme `file` is not allowed"), + "{described}" + ); + assert!(!described.contains("iii worker add"), "{described}"); + } +} diff --git a/document/src/config.rs b/document/src/config.rs new file mode 100644 index 000000000..2df480cf2 --- /dev/null +++ b/document/src/config.rs @@ -0,0 +1,396 @@ +//! Operator-facing runtime configuration. +//! +//! The authoritative value comes from the `configuration` worker at boot +//! (see [`crate::configuration`]); a `--config` YAML file, when passed, only +//! SEEDS the initial registration. Every field has a serde default so an empty +//! object yields a fully-populated config, and every field is a per-call +//! tuning knob read from the live snapshot — nothing here requires a restart. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Root config shape. Unknown keys are rejected so a typo'd field fails loudly +/// instead of silently running the default. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WorkerConfig { + /// Largest document accepted, in bytes. Guards against a path or a base64 + /// blob large enough to exhaust memory during parsing. + #[serde(default = "default_max_input_bytes")] + pub max_input_bytes: u64, + + /// Default cap on the characters of markdown returned in one response. A + /// capped response still reports the true total, so the caller knows what + /// it did not receive. Per-call `max_chars` overrides this; `0` means no + /// cap. + #[serde(default = "default_max_chars")] + pub max_chars: usize, + + /// Characters of leading content included as a preview alongside a capped + /// body. + #[serde(default = "default_preview_chars")] + pub preview_chars: usize, + + /// Largest number of embedded assets `document::extract-assets` returns in + /// one response. A slide deck carries one image per slide, and a long one + /// would otherwise return hundreds. + #[serde(default = "default_max_assets")] + pub max_assets: usize, + + /// Largest single asset returned with its bytes, in bytes. A larger asset + /// is still listed with its type and size — the caller learns it exists + /// and can decide — but its payload is left out rather than base64'd into + /// a response nobody can use. + #[serde(default = "default_max_asset_bytes")] + pub max_asset_bytes: u64, + + /// Total bytes of asset payload one `document::extract-assets` response may + /// carry. + /// + /// The per-asset ceiling alone is not a bound on the response: 24 assets of + /// 8 MiB each is roughly a quarter of a gigabyte once base64 inflates it, + /// which is not a response any caller wanted. Encoding stops when this + /// budget is spent and the response says it was truncated. + #[serde(default = "default_max_assets_total_bytes")] + pub max_assets_total_bytes: u64, + + /// Vision model `document::ocr` reads pages with when a call names none. + /// Unset means every call has to choose, which is the safer default for a + /// function that spends money per page. + #[serde(default)] + pub ocr_model: Option, + + /// Pages `document::ocr` transcribes in one call. The ceiling is a spend + /// limit, not a technical one: a caller that passes `pages` decides for + /// itself, and a caller that does not should not accidentally read a + /// four-hundred-page scan. + #[serde(default = "default_max_ocr_pages")] + pub max_ocr_pages: usize, + + /// Budget for one bus call `document::ocr` makes — rendering a page or + /// reading it. Rendering starts a browser and a vision model on a long page + /// is slow, so this is generous next to the other limits here. + #[serde(default = "default_ocr_timeout_ms")] + pub ocr_timeout_ms: u64, + + /// Milliseconds to let a rendered page paint before it is captured. + /// + /// `browser::navigate` returns on the load event, which for a PDF fires + /// when the viewer has loaded — not when it has drawn the page. Capturing + /// on that signal alone photographs an empty viewer, and the model dutifully + /// reports a blank image. + #[serde(default = "default_render_settle_ms")] + pub ocr_render_settle_ms: u64, + + /// Cache page transcriptions in the `state` worker, keyed by document + /// content and page. Re-reading the same scan then costs nothing. Turn it + /// off for a rig with no `state` worker, or when transcriptions should + /// never be persisted. + #[serde(default = "default_true")] + pub ocr_cache: bool, +} + +fn default_max_input_bytes() -> u64 { + 64 * 1024 * 1024 +} + +fn default_max_chars() -> usize { + 40_000 +} + +fn default_preview_chars() -> usize { + 600 +} + +fn default_max_assets() -> usize { + 24 +} + +fn default_max_asset_bytes() -> u64 { + 8 * 1024 * 1024 +} + +fn default_max_assets_total_bytes() -> u64 { + 32 * 1024 * 1024 +} + +fn default_max_ocr_pages() -> usize { + 20 +} + +fn default_ocr_timeout_ms() -> u64 { + 120_000 +} + +fn default_render_settle_ms() -> u64 { + 2_000 +} + +fn default_true() -> bool { + true +} + +impl Default for WorkerConfig { + fn default() -> Self { + Self { + max_input_bytes: default_max_input_bytes(), + max_chars: default_max_chars(), + preview_chars: default_preview_chars(), + max_assets: default_max_assets(), + max_asset_bytes: default_max_asset_bytes(), + max_assets_total_bytes: default_max_assets_total_bytes(), + ocr_model: None, + max_ocr_pages: default_max_ocr_pages(), + ocr_timeout_ms: default_ocr_timeout_ms(), + ocr_render_settle_ms: default_render_settle_ms(), + ocr_cache: default_true(), + } + } +} + +impl WorkerConfig { + /// Parse a seed config from YAML, expanding `${NAME}` against the process + /// env FIRST (the seed file is the only path that needs expansion — values + /// fetched from `configuration::get` are already env-expanded by the + /// configuration worker), then deserializing. + pub fn from_yaml(yaml: &str) -> Result { + let expanded = expand_env(yaml); + let parsed: Self = + serde_yaml::from_str(&expanded).map_err(|e| format!("yaml parse: {e}"))?; + parsed.validate() + } + + /// Reject values that parse but cannot mean anything. + /// + /// A zero asset ceiling is the interesting case: `max_assets: 0` would make + /// `document::extract-assets` always return nothing while reporting + /// success, which reads as "this deck has no images" rather than as a + /// misconfiguration. + fn validate(self) -> Result { + if self.max_assets == 0 { + return Err( + "max_assets must be at least 1; a ceiling of 0 makes every extraction look like \ + an empty document" + .to_string(), + ); + } + if self.max_asset_bytes == 0 { + return Err( + "max_asset_bytes must be at least 1; a ceiling of 0 drops the bytes of every asset" + .to_string(), + ); + } + if self.max_assets_total_bytes == 0 { + return Err( + "max_assets_total_bytes must be at least 1; a budget of 0 returns every asset \ + without its bytes while reporting success" + .to_string(), + ); + } + if self.max_ocr_pages == 0 { + return Err( + "max_ocr_pages must be at least 1; a ceiling of 0 makes document::ocr transcribe \ + nothing while reporting success" + .to_string(), + ); + } + Ok(self) + } + + /// Read and parse a YAML seed file (env-expanded — see [`Self::from_yaml`]). + pub fn from_file(path: &str) -> Result { + let raw = std::fs::read_to_string(path).map_err(|e| format!("read {path}: {e}"))?; + Self::from_yaml(&raw) + } + + /// Parse a config from a JSON value already env-expanded by the + /// configuration worker. Does NOT run [`expand_env`] (double expansion + /// would be a bug) and tolerates a zero-field object (serde defaults fill + /// in). + pub fn from_json(value: &Value) -> Result { + let parsed: Self = + serde_json::from_value(value.clone()).map_err(|e| format!("json parse: {e}"))?; + parsed.validate() + } + + pub fn to_json(&self) -> Value { + serde_json::to_value(self).expect("WorkerConfig serializes") + } + + /// The JSON Schema registered with the `configuration` worker. Field + /// doc-comments become property descriptions; the shipped defaults are + /// attached as a top-level `example`. + pub fn json_schema() -> Value { + let root = schemars::schema_for!(WorkerConfig); + let mut schema = + serde_json::to_value(&root.schema).expect("WorkerConfig JSON Schema serializes"); + if let Some(obj) = schema.as_object_mut() { + if !root.definitions.is_empty() { + obj.insert( + "definitions".into(), + serde_json::to_value(&root.definitions).expect("definitions serialize"), + ); + } + obj.insert("example".into(), WorkerConfig::default().to_json()); + } + schema + } + + /// Effective character cap for one response: the per-call override when + /// present, else the configured default. `0` means uncapped. + pub fn effective_max_chars(&self, requested: Option) -> usize { + requested.unwrap_or(self.max_chars) + } + + /// Effective asset ceiling for one response: a per-call request narrows the + /// configured ceiling and can never lift it. + /// + /// `Some(0)` means zero, not "use the default". The schema documents the + /// field as narrowing, so quietly widening a request for none into a + /// request for all of them hands back bytes the caller asked not to + /// receive; `null` is how a caller says it has no opinion. + pub fn effective_max_assets(&self, requested: Option) -> usize { + match requested { + Some(n) => n.min(self.max_assets), + None => self.max_assets, + } + } +} + +/// Expand `${NAME}` and `${NAME:default}` against the process env. An unset +/// variable with no default expands to the empty string, matching the +/// configuration worker's own expansion. +fn expand_env(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let mut rest = input; + while let Some(start) = rest.find("${") { + out.push_str(&rest[..start]); + let after = &rest[start + 2..]; + match after.find('}') { + Some(end) => { + let spec = &after[..end]; + let (name, fallback) = match spec.split_once(':') { + Some((n, d)) => (n, Some(d)), + None => (spec, None), + }; + match (std::env::var(name), fallback) { + (Ok(v), _) => out.push_str(&v), + (Err(_), Some(d)) => out.push_str(d), + (Err(_), None) => { + tracing::warn!(var = %name, "config references undefined env var") + } + } + rest = &after[end + 1..]; + } + None => { + out.push_str("${"); + rest = after; + } + } + } + out.push_str(rest); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_yaml_yields_defaults() { + let cfg = WorkerConfig::from_yaml("{}").expect("empty object parses"); + assert_eq!(cfg, WorkerConfig::default()); + } + + #[test] + fn yaml_overrides_each_field() { + let cfg = WorkerConfig::from_yaml( + "max_input_bytes: 1024\n\ + max_chars: 10\n\ + preview_chars: 5\n\ + max_assets: 3\n\ + max_asset_bytes: 2048\n", + ) + .expect("full object parses"); + assert_eq!(cfg.max_input_bytes, 1024); + assert_eq!(cfg.max_chars, 10); + assert_eq!(cfg.preview_chars, 5); + assert_eq!(cfg.max_assets, 3); + assert_eq!(cfg.max_asset_bytes, 2048); + } + + #[test] + fn unknown_field_is_rejected() { + let err = WorkerConfig::from_yaml("max_charz: 10\n").expect_err("typo must fail loudly"); + assert!( + err.contains("max_charz"), + "error should name the field: {err}" + ); + } + + /// A zero ceiling reads as an empty document rather than as a broken + /// config, so it is refused on both parse paths. + #[test] + fn zero_ceilings_are_rejected() { + let err = WorkerConfig::from_yaml("max_assets: 0\n").expect_err("zero assets"); + assert!(err.contains("max_assets"), "{err}"); + + let err = WorkerConfig::from_json(&serde_json::json!({ "max_asset_bytes": 0 })) + .expect_err("zero asset bytes"); + assert!(err.contains("max_asset_bytes"), "{err}"); + } + + #[test] + fn json_round_trips() { + let cfg = WorkerConfig { + max_chars: 123, + ..WorkerConfig::default() + }; + let back = WorkerConfig::from_json(&cfg.to_json()).expect("round trip"); + assert_eq!(cfg, back); + } + + #[test] + fn schema_carries_defaults_as_example() { + let schema = WorkerConfig::json_schema(); + assert_eq!(schema["example"], WorkerConfig::default().to_json()); + assert!(schema["properties"]["max_chars"]["description"].is_string()); + } + + #[test] + fn per_call_max_chars_overrides_the_default() { + let cfg = WorkerConfig::default(); + assert_eq!(cfg.effective_max_chars(None), cfg.max_chars); + assert_eq!(cfg.effective_max_chars(Some(7)), 7); + assert_eq!(cfg.effective_max_chars(Some(0)), 0); + } + + /// A per-call asset request narrows the operator's ceiling and never lifts + /// it: the limit exists to bound one response, and a caller asking for a + /// thousand images is exactly the case it is there for. + #[test] + fn a_call_cannot_raise_the_asset_ceiling() { + let cfg = WorkerConfig { + max_assets: 5, + ..WorkerConfig::default() + }; + assert_eq!(cfg.effective_max_assets(None), 5); + assert_eq!(cfg.effective_max_assets(Some(2)), 2); + assert_eq!(cfg.effective_max_assets(Some(500)), 5); + // Zero is an answer, not an absent opinion. + assert_eq!(cfg.effective_max_assets(Some(0)), 0); + } + + #[test] + fn env_expansion_applies_to_the_seed_only() { + std::env::set_var("DOCUMENT_TEST_CHARS", "99"); + let cfg = WorkerConfig::from_yaml("max_chars: ${DOCUMENT_TEST_CHARS}\n").expect("expands"); + assert_eq!(cfg.max_chars, 99); + std::env::remove_var("DOCUMENT_TEST_CHARS"); + + let cfg = + WorkerConfig::from_yaml("max_chars: ${DOCUMENT_UNSET_VAR:42}\n").expect("falls back"); + assert_eq!(cfg.max_chars, 42); + } +} diff --git a/document/src/configuration.rs b/document/src/configuration.rs new file mode 100644 index 000000000..2e3e0cfcb --- /dev/null +++ b/document/src/configuration.rs @@ -0,0 +1,260 @@ +//! Integration with the `configuration` worker: register the schema, fetch the +//! authoritative value at boot, and hot-reload it when it changes. +//! +//! Every field here is a per-call tuning knob read from the live snapshot, so +//! there is nothing structural to rebuild and nothing that needs a restart. +//! +//! `configuration` is a REQUIRED boot dependency: a failed register or fetch +//! aborts startup rather than running on a guessed size ceiling. + +use std::sync::Arc; +use std::time::Duration; + +use iii_sdk::errors::Error; +use iii_sdk::protocol::{RegisterTriggerInput, TriggerRequest}; +use iii_sdk::{IIIClient, RegisterFunction}; +use serde_json::{json, Value}; +use tokio::sync::RwLock; + +use crate::config::WorkerConfig; + +/// Hot-swappable config snapshot shared with every handler. A handler takes a +/// `read().await`, clones the inner `Arc` out, and drops the lock before doing +/// any work; `apply_config` replaces the inner `Arc` under the write lock. +pub type ConfigCell = Arc>>; + +pub const CONFIG_ID: &str = "document"; +const CONFIG_FN_ID: &str = "document::on-config-change"; +const CONFIG_RETRIES: u32 = 3; +/// Base backoff between configuration RPC retries, multiplied by the attempt +/// number for a linear backoff. +const CONFIG_RETRY_BACKOFF_MS: u64 = 250; + +/// Register this worker's configuration schema. When `seed` is present its +/// value becomes `initial_value`; otherwise the built-in default is seeded only +/// when nothing is stored yet, so calling this every boot is safe. +pub async fn register_config(iii: &IIIClient, seed: Option<&WorkerConfig>) -> Result<(), String> { + let mut payload = json!({ + "id": CONFIG_ID, + "name": "Document", + "description": "Limits for converting documents to markdown: the size ceiling on an \ + accepted file, the cap on how much markdown one response returns, and how \ + many embedded images an extraction hands back.", + "schema": WorkerConfig::json_schema(), + }); + if let Some(seed) = seed { + payload["initial_value"] = seed.to_json(); + } else if should_seed_default_value(iii).await? { + payload["initial_value"] = WorkerConfig::default().to_json(); + } + trigger_with_retry(iii, "configuration::register", payload).await?; + Ok(()) +} + +/// Read the live configuration (env-expanded by the configuration worker; +/// `from_json` does NOT re-expand). +pub async fn fetch_config(iii: &IIIClient) -> Result { + let value = get_config_value(iii).await?; + if value.is_null() { + tracing::info!("no configuration value found; using built-in defaults"); + return Ok(WorkerConfig::default()); + } + WorkerConfig::from_json(&value) +} + +async fn should_seed_default_value(iii: &IIIClient) -> Result { + match try_get_config_value(iii).await? { + None => Ok(true), + Some(value) if value.is_null() => Ok(true), + Some(_) => Ok(false), + } +} + +async fn get_config_value(iii: &IIIClient) -> Result { + try_get_config_value(iii) + .await? + .ok_or_else(|| format!("configuration `{CONFIG_ID}` not found")) +} + +/// `Ok(None)` when the entry does not exist. The engine's missing-entry codes +/// vary in case, so match case-insensitively. +async fn try_get_config_value(iii: &IIIClient) -> Result, String> { + match trigger_with_retry(iii, "configuration::get", json!({ "id": CONFIG_ID })).await { + Ok(resp) => Ok(resp.get("value").cloned()), + Err(e) if e.to_ascii_uppercase().contains("NOT_FOUND") => Ok(None), + Err(e) => Err(e), + } +} + +/// Swap the config snapshot under the write lock. +pub async fn apply_config(cell: &ConfigCell, cfg: WorkerConfig) { + *cell.write().await = Arc::new(cfg); +} + +/// Payload of the internal config-change handler. The handler re-fetches the +/// authoritative value, so this carries only the advisory id; a struct rather +/// than a `Value` keeps the request schema concrete. +#[derive(Debug, Default, serde::Deserialize, schemars::JsonSchema)] +pub struct OnConfigChangeEvent { + /// Configuration id that changed (advisory; the handler re-fetches). + #[serde(default)] + pub id: Option, +} + +/// Ack returned by the internal config-change handler. +#[derive(Debug, serde::Serialize, schemars::JsonSchema)] +pub struct OnConfigChangeResponse { + pub ok: bool, +} + +/// Register the internal config-change handler and bind a `configuration` +/// trigger. The handler re-fetches via `configuration::get` and ignores the +/// trigger payload, so a direct call can never inject config. +pub fn register_config_trigger(iii: &IIIClient, cell: ConfigCell) -> Result<(), Error> { + let cell_for_fn = cell.clone(); + let engine = iii.clone(); + iii.register_function( + CONFIG_FN_ID, + RegisterFunction::new_async(move |_event: OnConfigChangeEvent| { + let cell = cell_for_fn.clone(); + let engine = engine.clone(); + async move { + on_config_change(&engine, &cell).await; + Ok::(OnConfigChangeResponse { ok: true }) + } + }) + .description( + "Internal: hot-reload the document worker from the authoritative configuration when \ + it changes, swapping the per-call snapshot.", + ), + ); + + iii.register_trigger(RegisterTriggerInput { + trigger_type: "configuration".to_string(), + function_id: CONFIG_FN_ID.to_string(), + config: json!({ + "configuration_id": CONFIG_ID, + "event_types": ["configuration:updated"], + }), + metadata: None, + })?; + Ok(()) +} + +/// Reload from the AUTHORITATIVE configuration. +/// +/// The caller-supplied trigger payload is deliberately ignored: +/// `document::on-config-change` is a bus function, so trusting a `new_value` in +/// the payload would let any caller lift the size ceiling without touching +/// persisted state. +async fn on_config_change(iii: &IIIClient, cell: &ConfigCell) { + let cfg = match fetch_config(iii).await { + Ok(cfg) => cfg, + Err(e) => { + tracing::error!( + error = %e, + "config-change: failed to fetch authoritative configuration; keeping previous config" + ); + return; + } + }; + apply_config(cell, cfg).await; + tracing::info!("document configuration reloaded"); +} + +/// `true` for the one error that is an answer rather than a failure: the entry +/// does not exist yet. Retrying it wastes the backoff on every first boot and +/// logs two warnings for a completely normal state. +fn is_not_found(error: &str) -> bool { + error.to_ascii_uppercase().contains("NOT_FOUND") +} + +async fn trigger_with_retry( + iii: &IIIClient, + function_id: &str, + payload: Value, +) -> Result { + let mut last_err = String::new(); + for attempt in 1..=CONFIG_RETRIES { + match iii + .trigger(TriggerRequest { + function_id: function_id.to_string(), + payload: payload.clone(), + action: None, + timeout_ms: None, + }) + .await + { + Ok(v) => return Ok(v), + Err(e) => { + last_err = e.to_string(); + if is_not_found(&last_err) { + return Err(last_err); + } + if attempt < CONFIG_RETRIES { + tracing::warn!( + function_id, + attempt, + error = %last_err, + "configuration RPC failed; retrying" + ); + tokio::time::sleep(Duration::from_millis( + CONFIG_RETRY_BACKOFF_MS * u64::from(attempt), + )) + .await; + } + } + } + } + Err(format!( + "{function_id} failed after {CONFIG_RETRIES} attempts: {last_err}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A missing entry is the normal first-boot state, not a transient + /// failure. Retrying it spends the whole backoff and logs warnings on + /// every clean install. + #[test] + fn a_missing_entry_is_not_retried() { + assert!(is_not_found( + "remote error (NOT_FOUND): configuration 'document' not found" + )); + assert!(is_not_found("STATEMENT_NOT_FOUND")); + assert!(!is_not_found("connection reset by peer")); + assert!(!is_not_found("timed out")); + } + + #[tokio::test] + async fn apply_config_swaps_the_snapshot() { + let cell: ConfigCell = Arc::new(RwLock::new(Arc::new(WorkerConfig::default()))); + assert_eq!( + cell.read().await.max_chars, + WorkerConfig::default().max_chars + ); + + apply_config( + &cell, + WorkerConfig { + max_chars: 7, + ..WorkerConfig::default() + }, + ) + .await; + assert_eq!(cell.read().await.max_chars, 7); + } + + /// The config-change handler must stay off the public catalog: it is + /// registered here, not in `functions::register_all`. + #[test] + fn the_reload_handler_is_not_on_the_public_catalog() { + let ids: Vec<&str> = crate::functions::catalog() + .iter() + .map(|s| s.function_id) + .collect(); + assert!(!ids.contains(&CONFIG_FN_ID)); + } +} diff --git a/document/src/format.rs b/document/src/format.rs new file mode 100644 index 000000000..9b47d9ec5 --- /dev/null +++ b/document/src/format.rs @@ -0,0 +1,293 @@ +//! The wire vocabulary for formats, and how it maps onto the converter. +//! +//! The converter's own `Format` is a Rust enum with no serde derives, so it +//! cannot be the wire type; this module owns the names a caller sees and the +//! translation in both directions. Keeping them separate is also what lets the +//! wire stay stable when the converter adds a variant. +//! +//! Two things ride along with the name because a caller needs them and would +//! otherwise hard-code a match of its own: the `family` a format belongs to +//! (what the document IS — prose, a sheet, a deck), and whether the format was +//! recognised from the bytes or only from the file extension. The distinction +//! matters: a mislabelled `.txt` that is really a Word file converts fine, and +//! a `.csv` cannot be recognised from content at all, so "detected from the +//! extension" is a weaker claim that a caller may want to act on. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// A format this worker converts. The names are the wire vocabulary: stable, +/// lowercase, and independent of the file extension that named them (`.docm` +/// is `docx`, `.xlsb` is `excel`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Format { + /// Binary Word 97-2003 (`.doc`). + Doc, + /// WordprocessingML (`.docx`, `.docm`). + Docx, + /// OpenDocument Text (`.odt`). + Odt, + /// Rich Text Format (`.rtf`). + Rtf, + /// Binary PowerPoint 97-2003 (`.ppt`, `.pps`, `.pot`). + Ppt, + /// PresentationML (`.pptx`, `.pptm`, `.ppsx`, `.ppsm`). + Pptx, + /// OpenDocument Presentation (`.odp`). + Odp, + /// Excel workbooks in every container (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`). + Excel, + /// OpenDocument Spreadsheet (`.ods`). + Ods, + /// Delimiter-separated text (`.csv`). + Csv, + /// EPUB 2 and 3 (`.epub`). + Epub, + /// Portable Document Format (`.pdf`). + Pdf, +} + +/// What the document is, rather than which program wrote it. +/// +/// A caller routing a mixed bag of attachments cares that a file is a +/// spreadsheet, not that it is `.ods` rather than `.xlsx`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum Family { + /// Prose: Word, OpenDocument Text, RTF. + Prose, + /// Rows and columns: Excel, OpenDocument Spreadsheet, CSV. + Spreadsheet, + /// Slides: PowerPoint, OpenDocument Presentation. + Presentation, + /// A book: EPUB. + Book, + /// PDF, which is its own family because it is the one format with a + /// dedicated worker and a page-level OCR decision. + Pdf, +} + +/// How the format was arrived at, weakest claim last. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum DetectedFrom { + /// The caller named it, and the bytes were not consulted. + Requested, + /// The signature the format's specification designates (PDF header, RTF + /// open group, OLE stream names, ZIP package mimetype). + Content, + /// The file extension only. CSV carries no signature, so this is the only + /// way it is ever recognised; for any other format it means the content + /// did not match anything known. + Extension, +} + +impl Format { + /// The converter variant this name selects. + pub fn to_anydoc(self) -> anydoc::Format { + match self { + Format::Doc => anydoc::Format::Doc, + Format::Docx => anydoc::Format::Docx, + Format::Odt => anydoc::Format::Odt, + Format::Rtf => anydoc::Format::Rtf, + Format::Ppt => anydoc::Format::Ppt, + Format::Pptx => anydoc::Format::Pptx, + Format::Odp => anydoc::Format::Odp, + Format::Excel => anydoc::Format::Excel, + Format::Ods => anydoc::Format::Ods, + Format::Csv => anydoc::Format::Csv, + Format::Epub => anydoc::Format::Epub, + Format::Pdf => anydoc::Format::Pdf, + } + } + + /// The wire name for a converter variant. + pub fn from_anydoc(format: anydoc::Format) -> Self { + match format { + anydoc::Format::Doc => Format::Doc, + anydoc::Format::Docx => Format::Docx, + anydoc::Format::Odt => Format::Odt, + anydoc::Format::Rtf => Format::Rtf, + anydoc::Format::Ppt => Format::Ppt, + anydoc::Format::Pptx => Format::Pptx, + anydoc::Format::Odp => Format::Odp, + anydoc::Format::Excel => Format::Excel, + anydoc::Format::Ods => Format::Ods, + anydoc::Format::Csv => Format::Csv, + anydoc::Format::Epub => Format::Epub, + anydoc::Format::Pdf => Format::Pdf, + } + } + + pub fn family(self) -> Family { + match self { + Format::Doc | Format::Docx | Format::Odt | Format::Rtf => Family::Prose, + Format::Excel | Format::Ods | Format::Csv => Family::Spreadsheet, + Format::Ppt | Format::Pptx | Format::Odp => Family::Presentation, + Format::Epub => Family::Book, + Format::Pdf => Family::Pdf, + } + } + + /// `true` when the format parses into the document model, which is what + /// `document::extract-assets` walks. A PDF converts straight to markdown + /// and never builds a model, so it has no assets to hand back here. + pub fn has_document_model(self) -> bool { + self != Format::Pdf + } + + /// `true` when the format can embed an image or object at all. + /// + /// Counting assets costs a second parse of the whole document, and a CSV is + /// rows of text with nowhere to put a picture. Skipping it there is the + /// difference between one parse and two on the cheapest format people + /// attach. + pub fn carries_assets(self) -> bool { + self.has_document_model() && self != Format::Csv + } +} + +/// Resolve the format for a document, and say how the answer was reached. +/// +/// Order is deliberate: an explicit request wins because the caller may know +/// something the bytes do not say; content beats the extension because a +/// mislabelled file is common and a wrong extension is not worth failing over; +/// the extension is the last resort, and the only route for CSV. +/// [`resolve`], with the refusal a handler owes its caller when nothing +/// matched. +/// +/// `document::detect` wants the bare `Option` — "not a document I read" is its +/// answer, not a failure. Every function that goes on to convert wants the same +/// sentence, so it lives here rather than being written twice and drifting. +pub fn resolve_or_explain( + requested: Option, + bytes: &[u8], + file_name: Option<&str>, + label: &str, +) -> Result<(Format, DetectedFrom), String> { + resolve(requested, bytes, file_name).ok_or_else(|| { + format!( + "{label} is not a document this worker reads: nothing in its content matched a known \ + format, and its name did not name one either. Pass `format` if you know what it is." + ) + }) +} + +pub fn resolve( + requested: Option, + bytes: &[u8], + file_name: Option<&str>, +) -> Option<(Format, DetectedFrom)> { + if let Some(format) = requested { + return Some((format, DetectedFrom::Requested)); + } + if let Some(format) = anydoc::Format::from_bytes(bytes) { + return Some((Format::from_anydoc(format), DetectedFrom::Content)); + } + let extension = file_name + .and_then(|name| std::path::Path::new(name).extension()) + .and_then(|ext| ext.to_str())?; + anydoc::Format::from_extension(extension) + .map(|format| (Format::from_anydoc(format), DetectedFrom::Extension)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every wire name must survive the round trip through the converter's + /// enum. A missed arm here silently converts one format as another. + #[test] + fn every_format_round_trips_through_the_converter() { + for format in [ + Format::Doc, + Format::Docx, + Format::Odt, + Format::Rtf, + Format::Ppt, + Format::Pptx, + Format::Odp, + Format::Excel, + Format::Ods, + Format::Csv, + Format::Epub, + Format::Pdf, + ] { + assert_eq!(Format::from_anydoc(format.to_anydoc()), format); + } + } + + #[test] + fn wire_names_are_lowercase_and_stable() { + assert_eq!( + serde_json::to_string(&Format::Docx).expect("serializes"), + "\"docx\"" + ); + assert_eq!( + serde_json::to_string(&Format::Excel).expect("serializes"), + "\"excel\"" + ); + assert_eq!( + serde_json::to_string(&Family::Spreadsheet).expect("serializes"), + "\"spreadsheet\"" + ); + } + + /// A container variant is not its own wire name: `.docm` is a Word file + /// and `.xlsb` is a workbook, and a caller matching on the extension it + /// sent would otherwise have to know every alias. + #[test] + fn container_aliases_collapse_onto_one_name() { + let by_extension = |ext: &str| { + anydoc::Format::from_extension(ext) + .map(Format::from_anydoc) + .expect("known extension") + }; + assert_eq!(by_extension("docm"), Format::Docx); + assert_eq!(by_extension("xlsb"), Format::Excel); + assert_eq!(by_extension("ppsm"), Format::Pptx); + } + + #[test] + fn an_explicit_format_skips_detection() { + let (format, how) = resolve(Some(Format::Csv), b"not,a,csv,signature", None) + .expect("an explicit format always resolves"); + assert_eq!(format, Format::Csv); + assert_eq!(how, DetectedFrom::Requested); + } + + /// The PDF header is the cheapest real signature to assert against, and it + /// proves detection runs on content before the extension is consulted. + #[test] + fn content_beats_a_lying_extension() { + let (format, how) = + resolve(None, b"%PDF-1.7\n", Some("report.docx")).expect("content is recognised"); + assert_eq!(format, Format::Pdf); + assert_eq!(how, DetectedFrom::Content); + } + + /// CSV carries no signature. Without the extension fallback a spreadsheet + /// export would be unreadable, which is the common case for exported data. + #[test] + fn a_signature_less_format_falls_back_to_the_extension() { + let (format, how) = + resolve(None, b"a,b,c\n1,2,3\n", Some("rows.csv")).expect("extension resolves it"); + assert_eq!(format, Format::Csv); + assert_eq!(how, DetectedFrom::Extension); + } + + #[test] + fn nothing_recognisable_resolves_to_nothing() { + assert!(resolve(None, b"\x00\x01\x02", Some("mystery.bin")).is_none()); + assert!(resolve(None, b"\x00\x01\x02", None).is_none()); + } + + /// The document model is what `document::extract-assets` walks, and the + /// converter has no model form for a PDF. + #[test] + fn pdf_has_no_document_model() { + assert!(!Format::Pdf.has_document_model()); + assert!(Format::Pptx.has_document_model()); + } +} diff --git a/document/src/functions/assets.rs b/document/src/functions/assets.rs new file mode 100644 index 000000000..0dd31e874 --- /dev/null +++ b/document/src/functions/assets.rs @@ -0,0 +1,258 @@ +//! `document::extract-assets` — the pictures inside a document. +//! +//! Markdown renders an embedded image as its alt text, which is the right +//! default for text but throws away the one thing a slide deck is often made +//! of. A deck whose content is diagrams converts to a page of titles and reads +//! as an empty document; the pictures are the content, and a model that can see +//! images can use them. +//! +//! Two ceilings apply, and both report what they dropped rather than trimming +//! silently. `max_assets` bounds how many come back at all. `max_asset_bytes` +//! bounds the payload of one: a larger asset is still listed with its type and +//! size, so a caller learns it exists and can go read the file itself, but its +//! bytes are left out rather than base64'd into a response nobody can hold. + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::format::{self, Format}; +use crate::source::{describe_error, DocumentSource}; + +pub const ID: &str = "document::extract-assets"; +pub const DESC: &str = "Pull the images and embedded objects out of a document as base64, for a \ + deck or report whose content is pictures rather than text. Capped per \ + response and per asset; anything left out is still listed with its type \ + and size. Not available for PDFs — use pdf::extract-regions."; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: DocumentSource, + + /// Force a format instead of detecting one. + #[serde(default)] + pub format: Option, + + /// Assets to return in this response. Narrows the configured ceiling; it + /// cannot raise it. + #[serde(default)] + pub max_assets: Option, + + /// Return only assets whose media type starts with this, e.g. `image/`. + /// Omit for every asset. + #[serde(default)] + pub media_type_prefix: Option, + + /// Include the base64 payload. Set `false` to inventory a document — what + /// it holds and how big — without moving the bytes. + #[serde(default = "default_true")] + pub include_bytes: bool, +} + +fn default_true() -> bool { + true +} + +/// One embedded asset. `bytes_base64` is absent when the caller asked for an +/// inventory, or when this asset is over the per-asset ceiling — `omitted` +/// says which. +#[derive(Debug, Serialize, JsonSchema)] +pub struct Asset { + /// Position in the document's asset list, stable for a given document. + pub index: usize, + + /// MIME type, e.g. `image/png`. + pub media_type: String, + + /// The package part or stream it came from, for provenance. + pub origin_part: String, + + /// Size of the payload in bytes, whether or not the payload is included. + pub size_bytes: u64, + + /// The payload, base64-encoded. + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes_base64: Option, + + /// Why the payload is absent, when it is: `not_requested`, `too_large` + /// (this asset alone is over the per-asset ceiling), or `budget_spent` (the + /// response's total byte budget went on earlier assets — ask for this one + /// on its own). + #[serde(skip_serializing_if = "Option::is_none")] + pub omitted: Option<&'static str>, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// The format that was parsed. + pub format: Format, + + /// The assets, in document order, up to the effective ceiling. + pub assets: Vec, + + /// Assets the document holds after `media_type_prefix` is applied. Larger + /// than `assets.len()` when the ceiling cut the response short. + pub total_count: usize, + + /// `true` when the ceiling cut the response short. + pub truncated: bool, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time for the extraction. + pub elapsed_ms: u64, +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + + let file_name = req.source.file_name_hint(); + let (format, _) = format::resolve_or_explain( + req.format, + &bytes, + file_name.as_deref(), + &req.source.label(), + )?; + + // A PDF never builds a document model, so there is no asset list to walk. + // Say where the pictures actually live rather than returning an empty list + // that reads as "this document has none". + if !format.has_document_model() { + return Err( + "a PDF carries no extractable asset list here; use pdf::extract-regions to read a \ + region of a page, or pdf::classify to find the pages that are images" + .to_string(), + ); + } + + let document = anydoc::to_document(&bytes, format.to_anydoc()) + .map_err(|e| describe_error("asset extraction", &e))?; + + let prefix = req.media_type_prefix.as_deref(); + let matching: Vec<(usize, &anydoc::model::Asset)> = document + .assets + .iter() + .enumerate() + .filter(|(_, asset)| prefix.is_none_or(|p| asset.media_type.starts_with(p))) + .collect(); + + let ceiling = cfg.effective_max_assets(req.max_assets); + let total_count = matching.len(); + + // Two budgets, because the per-asset one alone does not bound a response: + // two dozen assets each just under the individual ceiling still add up to a + // payload nobody asked for. Spending the total stops the ENCODING, not the + // listing — every asset still comes back with its type and size, so the + // caller sees what exists and can ask for it directly. + let mut spent: u64 = 0; + let assets = matching + .into_iter() + .take(ceiling) + .map(|(index, asset)| { + let size_bytes = asset.bytes.len() as u64; + let (bytes_base64, omitted) = if !req.include_bytes { + (None, Some("not_requested")) + } else if size_bytes > cfg.max_asset_bytes { + (None, Some("too_large")) + } else if spent.saturating_add(size_bytes) > cfg.max_assets_total_bytes { + (None, Some("budget_spent")) + } else { + spent = spent.saturating_add(size_bytes); + (Some(BASE64.encode(&asset.bytes)), None) + }; + Asset { + index, + media_type: asset.media_type.clone(), + origin_part: asset.origin_part.clone(), + size_bytes, + bytes_base64, + omitted, + } + }) + .collect::>(); + + Ok(Response { + format, + truncated: total_count > assets.len(), + total_count, + assets, + source: req.source.label(), + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +#[cfg(test)] +mod tests { + use base64::engine::general_purpose::STANDARD as BASE64; + + use super::*; + + /// A PDF has no asset list here, and an empty list would read as "this + /// document holds no images", which is a different and wrong claim. + #[test] + fn a_pdf_is_refused_with_somewhere_to_go() { + let req = Request { + source: DocumentSource { + bytes_base64: Some(BASE64.encode(b"%PDF-1.7\n")), + file_name: Some("report.pdf".into()), + ..DocumentSource::default() + }, + format: None, + max_assets: None, + media_type_prefix: None, + include_bytes: true, + }; + let err = handle(req, &WorkerConfig::default()).expect_err("no model for a pdf"); + assert!(err.contains("pdf::extract-regions"), "{err}"); + } + + /// A CSV parses into a model with no assets at all, which is the honest + /// empty case: success, zero assets, nothing truncated. + #[test] + fn a_document_with_no_assets_returns_an_empty_list() { + let req = Request { + source: DocumentSource { + bytes_base64: Some(BASE64.encode(b"a,b\n1,2\n")), + file_name: Some("rows.csv".into()), + ..DocumentSource::default() + }, + format: None, + max_assets: None, + media_type_prefix: None, + include_bytes: true, + }; + let response = handle(req, &WorkerConfig::default()).expect("parses"); + assert!(response.assets.is_empty()); + assert_eq!(response.total_count, 0); + assert!(!response.truncated); + } + + #[test] + fn an_unrecognisable_file_says_to_name_the_format() { + let req = Request { + source: DocumentSource { + bytes_base64: Some(BASE64.encode(b"\x00\x01\x02")), + file_name: Some("mystery.bin".into()), + ..DocumentSource::default() + }, + format: None, + max_assets: None, + media_type_prefix: None, + include_bytes: true, + }; + let err = handle(req, &WorkerConfig::default()).expect_err("unrecognisable"); + assert!(err.contains("Pass `format`"), "{err}"); + } + + #[test] + fn include_bytes_defaults_to_true() { + let req: Request = serde_json::from_value(serde_json::json!({ "path": "deck.pptx" })) + .expect("minimal request parses"); + assert!(req.include_bytes); + } +} diff --git a/document/src/functions/detect.rs b/document/src/functions/detect.rs new file mode 100644 index 000000000..9e0c292ef --- /dev/null +++ b/document/src/functions/detect.rs @@ -0,0 +1,151 @@ +//! `document::detect` — what is this file, before anything tries to read it. +//! +//! Cheap on purpose: the signature lives in the first bytes of the file, so +//! this answers in microseconds where a conversion takes milliseconds. It is +//! what a caller holding a mixed bag of attachments runs first, to decide +//! whether a file is a document at all and which worker should read it. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::format::{self, DetectedFrom, Family, Format}; +use crate::source::DocumentSource; + +pub const ID: &str = "document::detect"; +pub const DESC: &str = "Identify a document's format from its bytes (falling back to the file \ + name for CSV, which carries no signature), and report which family it \ + belongs to and whether this worker can convert it. Microseconds, and no \ + conversion."; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: DocumentSource, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// The format, or `null` when nothing recognised it. A null means the file + /// is not one of the formats this worker reads — an image, an archive, a + /// plain text file — not that it is broken. + pub format: Option, + + /// What the document is: prose, a spreadsheet, a presentation, a book, a + /// PDF. Absent when the format is unknown. + #[serde(skip_serializing_if = "Option::is_none")] + pub family: Option, + + /// How the format was arrived at. `extension` is the weaker claim: the + /// content matched nothing known, and only the file name suggested this. + #[serde(skip_serializing_if = "Option::is_none")] + pub detected_from: Option, + + /// `true` when `document::to-markdown` can convert this file. + pub convertible: bool, + + /// `true` when the format can carry embedded assets for + /// `document::extract-assets` to pull out. False for a PDF, which converts + /// straight to markdown without a document model, and for a CSV, which is + /// rows of text with nowhere to put a picture. A caller routing on this + /// should not spend a call to be told a spreadsheet has no images. + pub has_assets: bool, + + /// Size of the document in bytes. + pub size_bytes: u64, + + /// Source label: the file name, or `` for an in-memory document + /// that arrived without one. + pub source: String, + + /// Wall-clock time for the detection. + pub elapsed_ms: u64, +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + + let resolved = format::resolve(None, &bytes, req.source.file_name_hint().as_deref()); + + Ok(Response { + format: resolved.map(|(format, _)| format), + family: resolved.map(|(format, _)| format.family()), + detected_from: resolved.map(|(_, how)| how), + convertible: resolved.is_some(), + has_assets: resolved.is_some_and(|(format, _)| format.carries_assets()), + size_bytes: bytes.len() as u64, + source: req.source.label(), + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +#[cfg(test)] +mod tests { + use base64::engine::general_purpose::STANDARD as BASE64; + use base64::Engine as _; + + use super::*; + + fn detect(bytes: &[u8], file_name: Option<&str>) -> Response { + let req = Request { + source: DocumentSource { + bytes_base64: Some(BASE64.encode(bytes)), + file_name: file_name.map(str::to_string), + ..DocumentSource::default() + }, + }; + handle(req, &WorkerConfig::default()).expect("detection never fails on readable bytes") + } + + #[test] + fn a_pdf_is_recognised_from_its_header() { + let response = detect(b"%PDF-1.7\n%\xE2\xE3\xCF\xD3\n", Some("report.pdf")); + assert_eq!(response.format, Some(Format::Pdf)); + assert_eq!(response.family, Some(Family::Pdf)); + assert_eq!(response.detected_from, Some(DetectedFrom::Content)); + assert!(response.convertible); + // A PDF has no document model, so it has no assets to extract here. + assert!(!response.has_assets); + } + + /// A CSV parses into a model, but rows of text cannot hold a picture. + /// Reporting `has_assets` for one sends a caller off to fetch an empty list. + #[test] + fn a_csv_reports_no_assets() { + let response = detect(b"name,total\nrohit,3\n", Some("rows.csv")); + assert_eq!(response.format, Some(Format::Csv)); + assert!(response.convertible); + assert!(!response.has_assets); + } + + #[test] + fn a_csv_is_recognised_from_its_name() { + let response = detect(b"name,total\nrohit,3\n", Some("rows.csv")); + assert_eq!(response.format, Some(Format::Csv)); + assert_eq!(response.family, Some(Family::Spreadsheet)); + assert_eq!(response.detected_from, Some(DetectedFrom::Extension)); + assert!(response.convertible); + } + + /// An unrecognised file is an answer, not a failure: the caller asked what + /// this is, and "not a document I read" is the answer. + #[test] + fn an_unknown_file_reports_itself_as_unconvertible() { + let response = detect(b"\x89PNG\r\n\x1a\n", Some("shot.png")); + assert_eq!(response.format, None); + assert!(response.family.is_none()); + assert!(!response.convertible); + assert!(!response.has_assets); + assert_eq!(response.size_bytes, 8); + } + + #[test] + fn a_missing_source_is_refused() { + let req = Request { + source: DocumentSource::default(), + }; + let err = handle(req, &WorkerConfig::default()).expect_err("no source"); + assert!(err.contains("provide a `path`"), "{err}"); + } +} diff --git a/document/src/functions/markdown.rs b/document/src/functions/markdown.rs new file mode 100644 index 000000000..0794c0b7b --- /dev/null +++ b/document/src/functions/markdown.rs @@ -0,0 +1,227 @@ +//! `document::to-markdown` — any office document as markdown that keeps its +//! shape. +//! +//! One serializer sits behind every format, so a `.doc` from 2003 and a `.pptx` +//! from yesterday come out with the same heading, table and list conventions. +//! That sameness is the point: a caller reading a mixed bag of attachments +//! writes one parser, not fourteen. +//! +//! The size cap is the other half of the job. A long report runs to hundreds of +//! thousands of characters, and handing that to a model wastes the context it +//! needed for the answer. Responses are capped by default and say so; a caller +//! that genuinely wants the whole document passes `max_chars: 0`, which is what +//! a worker-to-worker pipeline does when the document is going to storage +//! rather than to a model. +//! +//! PDFs convert here too, because the converter reads text-based ones already. +//! When the `pdf` worker is installed it is the better route for them: it +//! classifies scanned versus text-based, and reports which pages need OCR +//! rather than returning an empty document. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::WorkerConfig; +use crate::format::{self, DetectedFrom, Family, Format}; +use crate::source::{describe_error, Body, DocumentSource}; + +pub const ID: &str = "document::to-markdown"; +pub const DESC: &str = "Convert a Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV or PDF \ + document to markdown, preserving headings, lists, links and tables. The \ + format is detected from the bytes. Responses are capped; pass max_chars 0 \ + to take the whole document. For a PDF prefer pdf::classify first, which \ + reports which pages need OCR."; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: DocumentSource, + + /// Force a format instead of detecting one. Only needed when the content + /// carries no signature and the file name is absent or wrong. + #[serde(default)] + pub format: Option, + + /// Characters to return before truncating. Omit for the configured + /// default; `0` returns the whole document. + #[serde(default)] + pub max_chars: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// The format that was converted. + pub format: Format, + + /// What the document is: prose, a spreadsheet, a presentation, a book, a + /// PDF. + pub family: Family, + + /// How the format was arrived at. + pub detected_from: DetectedFrom, + + /// The markdown, capped per `max_chars`. + pub body: Body, + + /// Embedded images and objects the document carries. Their bytes are not + /// here — call `document::extract-assets` for those — but the count says + /// whether a deck's content is pictures rather than text, which markdown + /// alone would not reveal. + pub asset_count: usize, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time for the conversion. + pub elapsed_ms: u64, +} + +pub fn handle(req: Request, cfg: &WorkerConfig) -> Result { + let bytes = req.source.load(cfg)?; + let started = std::time::Instant::now(); + + let file_name = req.source.file_name_hint(); + let (format, detected_from) = format::resolve_or_explain( + req.format, + &bytes, + file_name.as_deref(), + &req.source.label(), + )?; + + let markdown = anydoc::to_markdown_bytes(&bytes, format.to_anydoc()) + .map_err(|e| describe_error("markdown conversion", &e))?; + + // The asset count comes from the document model, and reaching it costs a + // SECOND parse: the converter builds the model internally, renders it, and + // drops it, and its only other public entry point is the parse itself. That + // is the price of telling a caller a deck's content is pictures rather than + // letting it read as an empty document, so it is only paid for formats that + // can actually carry an asset. A failure here is not a reason to throw away + // markdown that converted fine. + let asset_count = if format.carries_assets() { + anydoc::to_document(&bytes, format.to_anydoc()) + .map(|doc| doc.assets.len()) + .unwrap_or(0) + } else { + 0 + }; + + let max_chars = cfg.effective_max_chars(req.max_chars); + let body = Body::new(markdown, max_chars, cfg.preview_chars); + + Ok(Response { + format, + family: format.family(), + detected_from, + body, + asset_count, + source: req.source.label(), + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +#[cfg(test)] +mod tests { + use base64::engine::general_purpose::STANDARD as BASE64; + use base64::Engine as _; + + use super::*; + + fn convert(bytes: &[u8], file_name: Option<&str>, max_chars: Option) -> Response { + let req = Request { + source: DocumentSource { + bytes_base64: Some(BASE64.encode(bytes)), + file_name: file_name.map(str::to_string), + ..DocumentSource::default() + }, + format: None, + max_chars, + }; + handle(req, &WorkerConfig::default()).expect("converts") + } + + /// CSV is the one format with no signature, so it exercises the whole + /// detection fallback as well as the conversion. + #[test] + fn a_csv_converts_to_a_markdown_table() { + let response = convert(b"name,total\nrohit,3\nsam,4\n", Some("rows.csv"), None); + assert_eq!(response.format, Format::Csv); + assert_eq!(response.family, Family::Spreadsheet); + assert_eq!(response.detected_from, DetectedFrom::Extension); + assert!( + response.body.text.contains("name"), + "{}", + response.body.text + ); + assert!( + response.body.text.contains("rohit"), + "{}", + response.body.text + ); + assert!(!response.body.truncated); + } + + /// The cap is what keeps a long document from eating the context the + /// question needed, and a truncated body has to say so. + #[test] + fn the_body_reports_truncation() { + let response = convert(b"name,total\nrohit,3\nsam,4\n", Some("rows.csv"), Some(8)); + assert!(response.body.truncated); + assert_eq!(response.body.chars, 8); + assert!(response.body.total_chars > 8); + assert!(response.body.preview.is_some()); + } + + #[test] + fn an_unrecognisable_file_says_to_name_the_format() { + let req = Request { + source: DocumentSource { + bytes_base64: Some(BASE64.encode(b"\x00\x01\x02\x03")), + file_name: Some("mystery.bin".into()), + ..DocumentSource::default() + }, + format: None, + max_chars: None, + }; + let err = handle(req, &WorkerConfig::default()).expect_err("unrecognisable"); + assert!(err.contains("Pass `format`"), "{err}"); + assert!(err.contains("mystery.bin"), "{err}"); + } + + /// A forced format skips detection, which is the escape hatch for a file + /// whose name and content both lie. + #[test] + fn an_explicit_format_overrides_detection() { + let req = Request { + source: DocumentSource { + bytes_base64: Some(BASE64.encode(b"a,b\n1,2\n")), + file_name: Some("data.txt".into()), + ..DocumentSource::default() + }, + format: Some(Format::Csv), + max_chars: None, + }; + let response = handle(req, &WorkerConfig::default()).expect("converts as csv"); + assert_eq!(response.detected_from, DetectedFrom::Requested); + assert!(response.body.text.contains('1')); + } + + /// A conversion failure has to arrive with the next move attached — an + /// agent that reads only "unsupported input" retries the same call. + #[test] + fn a_conversion_failure_carries_advice() { + let req = Request { + source: DocumentSource { + // A ZIP magic number with nothing inside it: recognised as a + // package, unusable as a document. + bytes_base64: Some(BASE64.encode(b"PK\x03\x04")), + file_name: Some("broken.docx".into()), + ..DocumentSource::default() + }, + format: Some(Format::Docx), + max_chars: None, + }; + let err = handle(req, &WorkerConfig::default()).expect_err("cannot convert"); + assert!(err.contains("markdown conversion failed"), "{err}"); + } +} diff --git a/document/src/functions/mod.rs b/document/src/functions/mod.rs new file mode 100644 index 000000000..3021ae20a --- /dev/null +++ b/document/src/functions/mod.rs @@ -0,0 +1,165 @@ +//! The worker's public surface: four functions over one converter. +//! +//! Three of the handlers are synchronous CPU work over an owned buffer, so each +//! runs on a blocking thread rather than on the async runtime. Conversion is +//! fast — single-digit milliseconds for a typical document — but a large +//! workbook is long enough to stall the executor and every other call sharing +//! it. +//! +//! `document::ocr` is the exception and registers differently: it spends its +//! time waiting on other workers rather than on this machine's CPU, so it stays +//! async and takes a [`Bus`] instead of a thread. + +pub mod assets; +pub mod detect; +pub mod markdown; +pub mod ocr; + +use std::sync::Arc; + +use iii_sdk::errors::Error; +use iii_sdk::{IIIClient, RegisterFunction}; + +use crate::bus::Bus; +use crate::configuration::ConfigCell; + +/// One entry of the wire surface: what a caller sees for one function. +pub struct FunctionSpec { + pub function_id: &'static str, + pub description: &'static str, + pub request_schema: schemars::schema::RootSchema, + pub response_schema: schemars::schema::RootSchema, +} + +/// Build a schema the same way iii-sdk does at registration, so the snapshot +/// equals what actually ships. +fn schema_of() -> schemars::schema::RootSchema { + schemars::r#gen::SchemaSettings::draft07() + .into_generator() + .into_root_schema_for::() +} + +fn spec(function_id: &'static str, description: &'static str) -> FunctionSpec +where + Req: schemars::JsonSchema, + Resp: schemars::JsonSchema, +{ + FunctionSpec { + function_id, + description, + request_schema: schema_of::(), + response_schema: schema_of::(), + } +} + +/// The full wire-surface catalog, in registration order. Golden-tested in +/// `tests/schemas.rs`; keep in lockstep with [`register_all`]. +pub fn catalog() -> Vec { + vec![ + spec::(detect::ID, detect::DESC), + spec::(markdown::ID, markdown::DESC), + spec::(assets::ID, assets::DESC), + spec::(ocr::ID, ocr::DESC), + ] +} + +/// Register one function whose handler is blocking CPU work over the live +/// config snapshot. +/// +/// The snapshot is read per call, so a configuration change takes effect on the +/// next invocation with no restart and no re-registration. +macro_rules! register_blocking { + ($iii:expr, $cell:expr, $module:ident) => {{ + let cell = $cell.clone(); + $iii.register_function( + $module::ID, + RegisterFunction::new_async(move |req: $module::Request| { + let cell = cell.clone(); + async move { + let cfg = cell.read().await.clone(); + tokio::task::spawn_blocking(move || $module::handle(req, &cfg)) + .await + .map_err(|e| Error::Handler(format!("{} panicked: {e}", $module::ID)))? + .map_err(Error::Handler) + } + }) + .description($module::DESC), + ); + }}; +} + +pub fn register_all(iii: &Arc, cell: &ConfigCell, bus: Arc) { + register_blocking!(iii, cell, detect); + register_blocking!(iii, cell, markdown); + register_blocking!(iii, cell, assets); + + // OCR waits on the browser and on a model rather than on this machine, so + // it stays on the async runtime: a blocking thread would sit idle for the + // whole call and there are only so many of them. + let cell = cell.clone(); + iii.register_function( + ocr::ID, + RegisterFunction::new_async(move |req: ocr::Request| { + let (cell, bus) = (cell.clone(), bus.clone()); + async move { + let cfg = cell.read().await.clone(); + ocr::handle(req, cfg, bus).await.map_err(Error::Handler) + } + }) + .description(ocr::DESC), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn catalog_lists_every_function_in_registration_order() { + let ids: Vec<&str> = catalog().iter().map(|s| s.function_id).collect(); + assert_eq!( + ids, + vec![ + "document::detect", + "document::to-markdown", + "document::extract-assets", + "document::ocr", + ] + ); + } + + /// Function ids are the public wire surface: kebab-case in multi-word + /// segments, never snake_case, and always under this worker's namespace. + #[test] + fn function_ids_follow_the_naming_rule() { + for spec in catalog() { + assert!( + spec.function_id.starts_with("document::"), + "{} is outside the worker namespace", + spec.function_id + ); + assert!( + !spec.function_id.contains('_'), + "{} uses snake_case; multi-word segments are kebab-case", + spec.function_id + ); + assert_eq!( + spec.function_id.to_lowercase(), + spec.function_id, + "{} is not lowercase", + spec.function_id + ); + } + } + + #[test] + fn every_function_carries_a_description() { + for spec in catalog() { + assert!( + spec.description.len() > 40, + "{} needs a description a caller can act on", + spec.function_id + ); + } + } +} diff --git a/document/src/functions/ocr.rs b/document/src/functions/ocr.rs new file mode 100644 index 000000000..fe76d6dd3 --- /dev/null +++ b/document/src/functions/ocr.rs @@ -0,0 +1,979 @@ +//! `document::ocr` — read a document nothing else here can read. +//! +//! Every other function in this worker walks a file's own structure. A scan has +//! none: it is pictures of text, and the characters exist only in the pixels. +//! This is the fallback branch of the same question the rest of the surface +//! answers, so it lives on the same worker rather than making a caller learn a +//! second one. +//! +//! Three inputs, one answer. An image goes straight to the model. A PDF is +//! rendered a page at a time by the `browser` worker, the only thing in the +//! fleet that turns a page into pixels. An office document whose text came back +//! empty has its embedded images pulled out and read the same way. +//! +//! Two rules shape the whole function, and both are about money. Nothing runs +//! implicitly: the attachment path reports a scan and names this function, and +//! an agent or a person decides to spend. And nothing is rendered before the +//! model is checked for vision, because a model that cannot see fails on the +//! first page after paying to produce it. + +use std::sync::Arc; + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use crate::bus::{describe_bus_failure, Bus}; +use crate::config::WorkerConfig; +use crate::format::{self, Format}; +use crate::source::{Body, DocumentSource}; + +pub const ID: &str = "document::ocr"; +pub const DESC: &str = "Transcribe a document that holds no readable text: a scanned PDF, a \ + photographed page, or a deck whose content is pictures. Renders the pages \ + that need it and reads them with a vision model, so it costs money per \ + page — pass `pages` (pdf::classify names them) to narrow it. Needs the \ + browser worker for PDFs and a vision model through llm-router."; + +/// The prompt every page is read with. +/// +/// Transcription, not description: a model told to "describe this image" writes +/// prose about a document, and the caller wanted the document. The instruction +/// to say nothing else is what keeps "Here is the text of the page:" out of the +/// markdown that ends up in someone's context. +const TRANSCRIBE_PROMPT: &str = "Transcribe every word of text in this image, in reading order, \ + as markdown. Preserve headings, lists and tables. Do not describe the image, do not \ + summarise, and do not add commentary: output only the transcription. If the image holds no \ + legible text at all, output nothing."; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct Request { + #[serde(flatten)] + pub source: DocumentSource, + + /// 1-indexed pages to transcribe, for a PDF. Omit for every page up to the + /// configured ceiling. This is the cost control: `pdf::classify` reports + /// which pages are scans, and passing that list keeps a long report from + /// being read a page at a time when only its cover is an image. + #[serde(default)] + pub pages: Option>, + + /// Vision model to read with. Omit for the configured default. The model is + /// checked for vision support before anything is rendered. + #[serde(default)] + pub model: Option, + + /// Characters to return before truncating. Omit for the configured + /// default; `0` returns everything transcribed. + #[serde(default)] + pub max_chars: Option, +} + +/// What one page turned into. +#[derive(Debug, Serialize, JsonSchema)] +pub struct PageText { + /// 1-indexed page number, or the asset's index for an office document. + pub page: u32, + /// The transcription. Empty when the page held no legible text. + pub text: String, + pub chars: usize, + /// `true` when this page came from the cache rather than the model. + pub cached: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + /// How the pixels were obtained: `image`, `pdf-render` or `document-assets`. + pub via: String, + + /// The joined transcription, capped per `max_chars`. + pub body: Body, + + /// Per-page transcriptions, in order. + pub pages: Vec, + + /// Pages actually read by the model this call. Excludes cache hits, so this + /// is what was paid for. + pub pages_transcribed: usize, + + /// Pages served from the cache, costing nothing. + pub pages_cached: usize, + + /// The model that read them. + pub model: String, + + /// Source label: the file name, or `` for an in-memory document. + pub source: String, + + /// Wall-clock time, rendering included. + pub elapsed_ms: u64, +} + +pub async fn handle( + req: Request, + cfg: Arc, + bus: Arc, +) -> Result { + let bytes = req.source.load(&cfg)?; + let started = std::time::Instant::now(); + let label = req.source.label(); + + let model = req + .model + .clone() + .or_else(|| cfg.ocr_model.clone()) + .ok_or_else(|| { + "no vision model chosen: pass `model`, or set `ocr_model` in this worker's \ + configuration. `router::models::list` reports which models support vision." + .to_string() + })?; + + // Before rendering anything. A model without vision fails on the first page + // AFTER the render has been paid for, and the error it returns says nothing + // about why. + ensure_vision(bus.as_ref(), &model, cfg.ocr_timeout_ms).await?; + + let route = route_for(&bytes, req.source.file_name_hint().as_deref(), &label)?; + let images = match &route { + RouteKind::Image(mime) => vec![PageImage { + page: 1, + mime: mime.clone(), + data: BASE64.encode(&bytes), + }], + RouteKind::Pdf => render_pdf(bus.as_ref(), &req, &cfg, &label).await?, + RouteKind::Assets(format) => asset_images(&bytes, *format, &cfg)?, + }; + + if images.is_empty() { + return Err(format!( + "{label} gave nothing to transcribe: no page was rendered and it carries no embedded \ + image" + )); + } + + let mut pages: Vec = Vec::with_capacity(images.len()); + let mut transcribed = 0usize; + let mut cached = 0usize; + for image in images { + let key = cache_key(&image.data, image.page, &model); + if let Some(hit) = cache_get(bus.as_ref(), &key, &cfg).await { + cached += 1; + pages.push(PageText { + page: image.page, + chars: hit.chars().count(), + text: hit, + cached: true, + }); + continue; + } + let text = transcribe(bus.as_ref(), &model, &image, &cfg).await?; + transcribed += 1; + cache_put(bus.as_ref(), &key, &text, &cfg).await; + pages.push(PageText { + page: image.page, + chars: text.chars().count(), + text, + cached: false, + }); + } + + let joined = pages + .iter() + .filter(|p| !p.text.trim().is_empty()) + .map(|p| p.text.trim()) + .collect::>() + .join("\n\n"); + let max_chars = cfg.effective_max_chars(req.max_chars); + + Ok(Response { + via: match route { + RouteKind::Image(_) => "image", + RouteKind::Pdf => "pdf-render", + RouteKind::Assets(_) => "document-assets", + } + .to_string(), + body: Body::new(joined, max_chars, cfg.preview_chars), + pages, + pages_transcribed: transcribed, + pages_cached: cached, + model, + source: label, + elapsed_ms: started.elapsed().as_millis() as u64, + }) +} + +/// One page's pixels on the way to the model. +struct PageImage { + page: u32, + mime: String, + data: String, +} + +/// Which of the three shapes this document is. +pub fn route_for(bytes: &[u8], file_name: Option<&str>, label: &str) -> Result { + if let Some(mime) = image_mime(bytes, file_name) { + return Ok(RouteKind::Image(mime)); + } + match format::resolve(None, bytes, file_name) { + Some((Format::Pdf, _)) => Ok(RouteKind::Pdf), + Some((format, _)) => Ok(RouteKind::Assets(format)), + None => Err(format!( + "{label} is neither an image nor a document this worker reads, so there is nothing to \ + transcribe" + )), + } +} + +/// Where the pixels come from for this document. +#[derive(Debug, PartialEq, Eq)] +pub enum RouteKind { + /// The file IS the image. + Image(String), + /// A PDF, rendered page by page through the browser worker. + Pdf, + /// An office document with no readable text; its embedded images are the + /// content. + Assets(Format), +} + +/// The image formats a vision model reads, recognised from the bytes. +/// +/// Signatures rather than the file name: an image pasted into a composer often +/// arrives named `image.png` whatever it actually is, and a wrong `mime` on the +/// wire is a provider error rather than a transcription. +pub fn image_mime(bytes: &[u8], file_name: Option<&str>) -> Option { + let mime = if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { + Some("image/png") + } else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + Some("image/jpeg") + } else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { + Some("image/gif") + } else if bytes.len() > 12 && &bytes[0..4] == b"RIFF" && &bytes[8..12] == b"WEBP" { + Some("image/webp") + } else { + None + }; + if let Some(mime) = mime { + return Some(mime.to_string()); + } + // A signature-less fallback for a caller that names the file: nothing here + // depends on it, but a `.jpg` whose header was stripped by a pipeline is a + // real thing to hit. + let ext = file_name + .and_then(|name| std::path::Path::new(name).extension()) + .and_then(|ext| ext.to_str()) + .map(|ext| ext.to_ascii_lowercase())?; + match ext.as_str() { + "png" => Some("image/png".to_string()), + "jpg" | "jpeg" => Some("image/jpeg".to_string()), + "gif" => Some("image/gif".to_string()), + "webp" => Some("image/webp".to_string()), + _ => None, + } +} + +/// Refuse a model that cannot see, before anything is rendered. +/// +/// The catalog is asked for its vision models rather than asked about this one: +/// `router::models::supports` needs the owning provider as well as the id, and +/// a caller naming a model rarely knows which provider serves it. The filtered +/// list answers without that. +/// +/// An id the catalog does not carry at all is NOT refused. The router fails +/// open on unknown models for the same reason: a model this worker has never +/// heard of is far more likely to be newer than the catalog than to be blind, +/// and refusing it would make the function unusable on a rig that is ahead. +async fn ensure_vision(bus: &dyn Bus, model: &str, timeout_ms: u64) -> Result<(), String> { + let seeing = bus + .trigger( + "router::models::list", + json!({ "capability": "vision" }), + timeout_ms, + ) + .await + .map_err(|e| describe_bus_failure("router::models::list", &e))?; + + if catalog_has(&seeing, model) { + return Ok(()); + } + + // Not among the models that see. Either it cannot, or the catalog does not + // know it — and those deserve different answers. + let everything = bus + .trigger("router::models::list", json!({}), timeout_ms) + .await + .map_err(|e| describe_bus_failure("router::models::list", &e))?; + + if catalog_has(&everything, model) { + Err(format!( + "{model} cannot read images, so it cannot transcribe anything. Pick a model whose \ + `supports_vision` is true in router::models::list." + )) + } else { + Ok(()) + } +} + +/// Whether a `router::models::list` answer carries this model. +/// +/// A model id reaches this worker in more than one shape: bare +/// (`claude-haiku-4-5`), or carrying the provider the console composes onto it +/// (`anthropic::claude-haiku-4-5`). Compare on the bare half of both sides. +pub fn catalog_has(answer: &Value, model: &str) -> bool { + let wanted = bare_model_id(model); + answer + .get("models") + .and_then(Value::as_array) + .is_some_and(|models| { + models + .iter() + .filter_map(|m| m.get("id").and_then(Value::as_str)) + .any(|id| bare_model_id(id) == wanted) + }) +} + +fn bare_model_id(model: &str) -> &str { + model.rsplit("::").next().unwrap_or(model) +} + +/// Render a PDF through the browser worker, one capture per page. +/// +/// Chromium is the only component in the fleet that rasterizes a page, and its +/// PDF viewer takes the page number in the fragment. The session is started and +/// stopped here rather than left open: a browser session is a whole Chrome +/// process, and holding one for the length of a transcription costs more than +/// re-navigating. +async fn render_pdf( + bus: &dyn Bus, + req: &Request, + cfg: &WorkerConfig, + label: &str, +) -> Result, String> { + let path = req.source.path.as_deref().ok_or_else(|| { + "rendering a PDF needs it on disk: pass `path` rather than `bytes_base64`, because the \ + browser opens the file by URL" + .to_string() + })?; + + let pages = match &req.pages { + Some(pages) if pages.is_empty() => { + return Err("`pages` was empty; omit it to read the whole document".to_string()) + } + Some(pages) => { + if pages.contains(&0) { + return Err("page numbers are 1-indexed; 0 is not a page".to_string()); + } + pages.clone() + } + None => (1..=cfg.max_ocr_pages as u32).collect(), + }; + let pages: Vec = pages.into_iter().take(cfg.max_ocr_pages).collect(); + + let started = bus + .trigger("browser::sessions::start", json!({}), cfg.ocr_timeout_ms) + .await + .map_err(|e| describe_bus_failure("browser::sessions::start", &e))?; + let session_id = started + .get("session_id") + .and_then(Value::as_str) + .ok_or_else(|| "browser::sessions::start returned no session_id".to_string())? + .to_string(); + + let rendered = render_pages(bus, &session_id, path, &pages, cfg).await; + + // Always stop the session, including on the failure path: a leaked Chrome + // process outlives the call that made it and counts against `max_sessions`. + let _ = bus + .trigger( + "browser::sessions::stop", + json!({ "session_id": session_id }), + cfg.ocr_timeout_ms, + ) + .await; + + let rendered = rendered?; + if rendered.is_empty() { + return Err(format!("{label}: no page could be rendered")); + } + Ok(rendered) +} + +async fn render_pages( + bus: &dyn Bus, + session_id: &str, + path: &str, + pages: &[u32], + cfg: &WorkerConfig, +) -> Result, String> { + let mut out = Vec::new(); + for &page in pages { + let url = format!("file://{path}#page={page}"); + bus.trigger( + "browser::navigate", + json!({ "session_id": session_id, "url": url }), + cfg.ocr_timeout_ms, + ) + .await + .map_err(|e| describe_navigate_failure(&e))?; + + // `navigate` returns on the load event, which for a PDF fires when the + // viewer has loaded rather than when it has drawn the page. Capturing + // on that signal photographs an empty viewer, and a model reading it + // reports a blank image — which is what the first live run produced. + if cfg.ocr_render_settle_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis(cfg.ocr_render_settle_ms)).await; + } + + let shot = bus + .trigger( + "browser::screenshot", + json!({ "session_id": session_id, "full_page": false }), + cfg.ocr_timeout_ms, + ) + .await + .map_err(|e| describe_bus_failure("browser::screenshot", &e))?; + + // A page past the end of the document renders as the last page again + // rather than failing, so a run with no explicit `pages` stops at the + // first repeat instead of transcribing the same page to the ceiling. + let Some((mime, data)) = image_from_screenshot(&shot) else { + break; + }; + if out.last().is_some_and(|last: &PageImage| last.data == data) { + out.pop(); + break; + } + out.push(PageImage { page, mime, data }); + } + Ok(out) +} + +/// The one browser refusal worth translating. +/// +/// A local PDF is opened over `file://`, and the browser worker ships with an +/// allowlist of `http` and `https` only. Its own error names the scheme but not +/// the setting, and "scheme `file` is not allowed" sends a reader looking +/// through this worker's configuration, where the answer is not. +fn describe_navigate_failure(err: &str) -> String { + if err.contains("scheme") && err.contains("file") { + return "the browser worker refuses `file://` URLs, so a local PDF cannot be rendered. Add \ + `file` to its allowed schemes: console workers tab, browser settings, Behavior, \ + Allowed URL schemes (or `allowed_schemes` in its configuration). It hot-applies." + .to_string(); + } + describe_bus_failure("browser::navigate", err) +} + +/// Pull the image block out of a `browser::screenshot` response. +pub fn image_from_screenshot(value: &Value) -> Option<(String, String)> { + let blocks = value.get("content")?.as_array()?; + for block in blocks { + let data = block.get("data").and_then(Value::as_str); + let mime = block.get("mime").and_then(Value::as_str); + if let (Some(mime), Some(data)) = (mime, data) { + if !data.is_empty() { + return Some((mime.to_string(), data.to_string())); + } + } + } + None +} + +/// The embedded images of a document whose text came back empty. +fn asset_images( + bytes: &[u8], + format: Format, + cfg: &WorkerConfig, +) -> Result, String> { + let document = anydoc::to_document(bytes, format.to_anydoc()) + .map_err(|e| crate::source::describe_error("reading embedded images", &e))?; + + Ok(document + .assets + .iter() + .filter(|asset| asset.media_type.starts_with("image/")) + .take(cfg.max_ocr_pages) + .enumerate() + .map(|(index, asset)| PageImage { + page: index as u32 + 1, + mime: asset.media_type.clone(), + data: BASE64.encode(&asset.bytes), + }) + .collect()) +} + +/// One page, read by the model. +async fn transcribe( + bus: &dyn Bus, + model: &str, + image: &PageImage, + cfg: &WorkerConfig, +) -> Result { + let answer = bus + .trigger( + "router::complete", + json!({ + "model": model, + "messages": [{ + "role": "user", + "content": [ + { "type": "text", "text": TRANSCRIBE_PROMPT }, + { "type": "image", "mime": image.mime, "data": image.data }, + ], + "timestamp": 0, + }], + }), + cfg.ocr_timeout_ms, + ) + .await + .map_err(|e| describe_bus_failure("router::complete", &e))?; + + Ok(text_of(&answer)) +} + +/// The text blocks of a `router::complete` answer, joined. +pub fn text_of(answer: &Value) -> String { + let Some(content) = answer + .get("message") + .and_then(|m| m.get("content")) + .and_then(Value::as_array) + else { + return String::new(); + }; + content + .iter() + .filter(|block| block.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|block| block.get("text").and_then(Value::as_str)) + .collect::>() + .join("") + .trim() + .to_string() +} + +/// Cache key for one page: the PIXELS that were read, and the model that read +/// them. +/// +/// Keying on the rendered image rather than on the source document is what +/// makes the cache self-correcting. A render that came out blank hashes +/// differently from the same page rendered properly, so fixing the renderer +/// invalidates every bad entry it produced instead of serving them forever — +/// which is exactly what happened the first time this ran against a real PDF. +/// It also means the same page arriving inside two different documents is +/// transcribed once. +/// +/// The trade is that a hit no longer skips the render, only the model call. +/// Rendering is a second of local Chromium; the model call is the money. +pub fn cache_key(image_data: &str, page: u32, model: &str) -> String { + format!("{}/{page}/{model}", content_hash(image_data.as_bytes())) +} + +/// FNV-1a over the document bytes. +/// +/// Not a cryptographic hash and does not need to be: this keys a cache of the +/// worker's own transcriptions, where a collision costs a wrong page of text +/// and nothing else. Hand-rolled to keep the dependency list at one crate. +fn content_hash(bytes: &[u8]) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in bytes { + hash ^= *byte as u64; + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("{hash:016x}") +} + +async fn cache_get(bus: &dyn Bus, key: &str, cfg: &WorkerConfig) -> Option { + if !cfg.ocr_cache { + return None; + } + let answer = bus + .trigger( + "state::get", + json!({ "scope": OCR_SCOPE, "key": key }), + cfg.ocr_timeout_ms, + ) + .await + .ok()?; + // `state::get` answers with the VALUE, so a stored string arrives as one. + answer + .as_str() + .map(str::to_string) + .or_else(|| { + answer + .get("value") + .and_then(Value::as_str) + .map(str::to_string) + }) + .filter(|text| !text.is_empty()) +} + +async fn cache_put(bus: &dyn Bus, key: &str, text: &str, cfg: &WorkerConfig) { + if !cfg.ocr_cache || text.trim().is_empty() { + return; + } + // Best effort: a cache that cannot be written is slower, not broken, and a + // rig with no `state` worker still transcribes. + let _ = bus + .trigger( + "state::set", + json!({ "scope": OCR_SCOPE, "key": key, "value": text }), + cfg.ocr_timeout_ms, + ) + .await; +} + +/// State scope for the transcription cache. Only page TEXT lives here; the +/// rendered images never leave the call that made them. +const OCR_SCOPE: &str = "document-ocr"; + +#[cfg(test)] +mod tests { + use super::*; + use crate::bus::test_bus::RecordedBus; + use base64::engine::general_purpose::STANDARD as B64; + + const PNG: &[u8] = b"\x89PNG\r\n\x1a\n\x00\x01"; + + fn cfg() -> Arc { + Arc::new(WorkerConfig { + ocr_model: Some("test-vision".into()), + ..WorkerConfig::default() + }) + } + + fn request(bytes: &[u8], name: &str) -> Request { + Request { + source: DocumentSource { + bytes_base64: Some(B64.encode(bytes)), + file_name: Some(name.to_string()), + ..DocumentSource::default() + }, + pages: None, + model: None, + max_chars: None, + } + } + + fn vision_ok(bus: RecordedBus) -> RecordedBus { + bus.on( + "router::models::list", + json!({ "models": [{ "id": "test-vision", "supports_vision": true }] }), + ) + } + + fn transcription(text: &str) -> Value { + json!({ + "message": { "role": "assistant", "content": [{ "type": "text", "text": text }] }, + "model": "test-vision", + "provider": "test", + }) + } + + #[tokio::test] + async fn an_image_goes_straight_to_the_model_with_no_browser() { + let bus = Arc::new( + vision_ok(RecordedBus::new()) + .on("router::complete", transcription("INVOICE 42")) + .on("state::set", json!({ "ok": true })), + ); + let response = handle(request(PNG, "receipt.png"), cfg(), bus.clone()) + .await + .expect("transcribes"); + + assert_eq!(response.via, "image"); + assert_eq!(response.body.text, "INVOICE 42"); + assert_eq!(response.pages_transcribed, 1); + assert!( + !bus.called().iter().any(|id| id.starts_with("browser::")), + "an image needs no rendering: {:?}", + bus.called() + ); + } + + /// The image travels as an image block, not as prose about one. + #[tokio::test] + async fn the_page_reaches_the_model_as_pixels() { + let bus = Arc::new( + vision_ok(RecordedBus::new()) + .on("router::complete", transcription("text")) + .on("state::set", json!({ "ok": true })), + ); + handle(request(PNG, "page.png"), cfg(), bus.clone()) + .await + .expect("transcribes"); + + let payload = &bus.payloads("router::complete")[0]; + let content = payload["messages"][0]["content"] + .as_array() + .expect("content"); + assert_eq!(content[1]["type"], "image"); + assert_eq!(content[1]["mime"], "image/png"); + assert!(content[1]["data"].as_str().is_some_and(|d| !d.is_empty())); + } + + /// Checking the model comes FIRST. A model that cannot see fails on the + /// first page otherwise, after the render has already been paid for. + #[tokio::test] + async fn a_model_without_vision_is_refused_before_anything_renders() { + // Present in the catalog, absent from the models that see. + let bus = Arc::new( + RecordedBus::new() + .on("router::models::list", json!({ "models": [] })) + .on( + "router::models::list", + json!({ "models": [{ "id": "test-vision" }] }), + ), + ); + let err = handle(request(PNG, "page.png"), cfg(), bus.clone()) + .await + .expect_err("refused"); + + assert!(err.contains("cannot read images"), "{err}"); + assert!( + !bus.called().iter().any(|id| id == "router::complete"), + "nothing may be read: {:?}", + bus.called() + ); + } + + /// A model the catalog has never heard of is likelier to be newer than the + /// catalog than to be blind, so it is read rather than refused — the same + /// stance the router itself takes on unknown models. + #[tokio::test] + async fn a_model_the_catalog_does_not_know_still_transcribes() { + let bus = Arc::new( + RecordedBus::new() + .on("router::models::list", json!({ "models": [] })) + .on("router::complete", transcription("readable")) + .on("state::set", json!({ "ok": true })), + ); + let response = handle(request(PNG, "page.png"), cfg(), bus) + .await + .expect("transcribes"); + assert_eq!(response.body.text, "readable"); + } + + #[tokio::test] + async fn a_missing_model_says_where_to_find_one() { + let bare = Arc::new(WorkerConfig::default()); + let bus = Arc::new(RecordedBus::new()); + let err = handle(request(PNG, "page.png"), bare, bus) + .await + .expect_err("no model"); + assert!(err.contains("router::models::list"), "{err}"); + } + + /// The cache is keyed by content, so the same page never gets paid for + /// twice, and a hit must not reach the model at all. + #[tokio::test] + async fn a_cached_page_costs_nothing() { + let bus = + Arc::new(vision_ok(RecordedBus::new()).on("state::get", json!("cached transcription"))); + let response = handle(request(PNG, "page.png"), cfg(), bus.clone()) + .await + .expect("serves from cache"); + + assert_eq!(response.pages_cached, 1); + assert_eq!(response.pages_transcribed, 0); + assert_eq!(response.body.text, "cached transcription"); + assert!( + !bus.called().iter().any(|id| id == "router::complete"), + "a cache hit must not call the model: {:?}", + bus.called() + ); + } + + /// The rendered pixels are never stored: only the text is. + #[tokio::test] + async fn only_the_text_is_cached() { + let bus = Arc::new( + vision_ok(RecordedBus::new()) + .on("router::complete", transcription("page one")) + .on("state::set", json!({ "ok": true })), + ); + handle(request(PNG, "page.png"), cfg(), bus.clone()) + .await + .expect("transcribes"); + + let stored = &bus.payloads("state::set")[0]; + assert_eq!(stored["value"], "page one"); + let serialized = stored.to_string(); + assert!( + !serialized.contains(&B64.encode(PNG)), + "the page image must not reach the cache" + ); + } + + #[tokio::test] + async fn a_pdf_without_a_path_says_why() { + let bus = Arc::new(vision_ok(RecordedBus::new())); + let err = handle(request(b"%PDF-1.7\n", "scan.pdf"), cfg(), bus) + .await + .expect_err("needs a path"); + assert!(err.contains("`path`"), "{err}"); + } + + #[tokio::test] + async fn a_missing_browser_worker_says_what_to_install() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("scan.pdf"); + std::fs::write(&path, b"%PDF-1.7\n").expect("write"); + + let bus = Arc::new(vision_ok(RecordedBus::new())); + let req = Request { + source: DocumentSource { + path: Some(path.to_string_lossy().to_string()), + ..DocumentSource::default() + }, + pages: Some(vec![1]), + model: None, + max_chars: None, + }; + let err = handle(req, cfg(), bus).await.expect_err("no browser"); + assert!(err.contains("iii worker add browser"), "{err}"); + } + + #[tokio::test] + async fn a_pdf_page_is_rendered_then_read() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("scan.pdf"); + std::fs::write(&path, b"%PDF-1.7\n").expect("write"); + + let bus = Arc::new( + vision_ok(RecordedBus::new()) + .on("browser::sessions::start", json!({ "session_id": "s-1" })) + .on("browser::navigate", json!({ "ok": true })) + .on( + "browser::screenshot", + json!({ "content": [{ "type": "image", "mime": "image/jpeg", "data": "AAAA" }] }), + ) + .on("router::complete", transcription("PAGE ONE")) + .on("state::set", json!({ "ok": true })) + .on("browser::sessions::stop", json!({ "ok": true })), + ); + + let req = Request { + source: DocumentSource { + path: Some(path.to_string_lossy().to_string()), + ..DocumentSource::default() + }, + pages: Some(vec![1]), + model: None, + max_chars: None, + }; + let response = handle(req, cfg(), bus.clone()).await.expect("transcribes"); + + assert_eq!(response.via, "pdf-render"); + assert_eq!(response.body.text, "PAGE ONE"); + let order = bus.called(); + let navigate = order.iter().position(|id| id == "browser::navigate"); + let complete = order.iter().position(|id| id == "router::complete"); + assert!(navigate < complete, "render before read: {order:?}"); + assert!( + order.contains(&"browser::sessions::stop".to_string()), + "the session must be stopped: {order:?}" + ); + // The page number rides in the URL fragment, which is how Chrome's PDF + // viewer is told which page to show. + let url = bus.payloads("browser::navigate")[0]["url"] + .as_str() + .expect("url") + .to_string(); + assert!(url.starts_with("file://"), "{url}"); + assert!(url.ends_with("#page=1"), "{url}"); + } + + /// A Chrome session is a whole browser process; leaking one on the failure + /// path counts against `max_sessions` until the worker restarts. + #[tokio::test] + async fn the_browser_session_is_stopped_even_when_a_page_fails() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("scan.pdf"); + std::fs::write(&path, b"%PDF-1.7\n").expect("write"); + + let bus = Arc::new( + vision_ok(RecordedBus::new()) + .on("browser::sessions::start", json!({ "session_id": "s-1" })) + .failing("browser::navigate", "target closed") + .on("browser::sessions::stop", json!({ "ok": true })), + ); + let req = Request { + source: DocumentSource { + path: Some(path.to_string_lossy().to_string()), + ..DocumentSource::default() + }, + pages: Some(vec![1]), + model: None, + max_chars: None, + }; + let err = handle(req, cfg(), bus.clone()) + .await + .expect_err("render fails"); + + assert!(err.contains("target closed"), "{err}"); + assert!( + bus.called() + .contains(&"browser::sessions::stop".to_string()), + "{:?}", + bus.called() + ); + } + + #[test] + fn a_blocked_file_url_names_the_setting_to_change() { + // The browser's own message names the scheme but not the setting, and + // the setting lives on a different worker than the one being read. + let described = describe_navigate_failure("scheme `file` is not allowed"); + assert!(described.contains("allowed_schemes"), "{described}"); + assert!(describe_navigate_failure("target crashed").contains("target crashed")); + } + + #[test] + fn images_are_recognised_by_signature_first() { + assert_eq!(image_mime(PNG, None).as_deref(), Some("image/png")); + assert_eq!( + image_mime(&[0xFF, 0xD8, 0xFF, 0x00], None).as_deref(), + Some("image/jpeg") + ); + // A stripped header falls back to the name. + assert_eq!( + image_mime(b"\x00\x00", Some("photo.JPG")).as_deref(), + Some("image/jpeg") + ); + assert_eq!(image_mime(b"%PDF-1.7", Some("scan.pdf")), None); + } + + #[test] + fn routing_picks_the_cheapest_path_that_works() { + assert_eq!( + route_for(PNG, Some("page.png"), "page.png").expect("image"), + RouteKind::Image("image/png".to_string()) + ); + assert_eq!( + route_for(b"%PDF-1.7\n", Some("scan.pdf"), "scan.pdf").expect("pdf"), + RouteKind::Pdf + ); + let err = route_for(b"\x00\x01\x02", Some("mystery.bin"), "mystery.bin") + .expect_err("nothing to do"); + assert!(err.contains("nothing to transcribe"), "{err}"); + } + + /// Keying on the pixels is what makes a fixed renderer invalidate the bad + /// entries it produced, rather than serving them forever. + #[test] + fn the_cache_key_follows_the_pixels() { + let good = cache_key("rendered-page-bytes", 1, "m"); + assert_eq!(good, cache_key("rendered-page-bytes", 1, "m")); + assert_ne!(good, cache_key("blank-page-bytes", 1, "m")); + assert_ne!(good, cache_key("rendered-page-bytes", 2, "m")); + // A different model is a different answer, not a fresher one. + assert_ne!(good, cache_key("rendered-page-bytes", 1, "better-model")); + } + + #[test] + fn a_transcription_is_read_out_of_the_router_envelope() { + assert_eq!(text_of(&transcription("hello")), "hello"); + assert_eq!(text_of(&json!({})), ""); + } +} diff --git a/document/src/lib.rs b/document/src/lib.rs new file mode 100644 index 000000000..73eeb298f --- /dev/null +++ b/document/src/lib.rs @@ -0,0 +1,7 @@ +pub mod bus; +pub mod config; +pub mod configuration; +pub mod format; +pub mod functions; +pub mod manifest; +pub mod source; diff --git a/document/src/main.rs b/document/src/main.rs new file mode 100644 index 000000000..9552c2785 --- /dev/null +++ b/document/src/main.rs @@ -0,0 +1,141 @@ +//! The document worker: convert any office document to markdown, locally. +//! +//! Boot order, and why: +//! +//! 1. tracing, then the CLI +//! 2. `--manifest` prints and returns without connecting, because the registry +//! publish pipeline calls it and must not need an engine +//! 3. connect +//! 4. register and fetch the configuration — a required boot dependency, so a +//! failure here aborts rather than running on guessed limits +//! 5. register the functions +//! 6. bind the configuration trigger LAST, so its handler closes over fully +//! built state +//! 7. wait for a signal, then shut the SDK down cleanly + +use std::sync::Arc; + +use clap::Parser; +use iii_sdk::runtime::WorkerMetadata; +use iii_sdk::{register_worker, InitOptions}; +use tokio::sync::RwLock; +use tracing_subscriber::EnvFilter; + +use document::bus::EngineBus; +use document::config::WorkerConfig; +use document::configuration::ConfigCell; +use document::{configuration, functions, manifest}; + +#[derive(Parser, Debug)] +#[command(name = "document", about = manifest::DESCRIPTION)] +struct Cli { + /// Optional one-time seed for the configuration entry on first + /// registration. Never overwrites a stored value. + #[arg(long)] + config: Option, + + /// Engine websocket URL. + #[arg(long, env = "III_URL", default_value = "ws://127.0.0.1:49134")] + url: String, + + /// Print the registry manifest and exit. + #[arg(long)] + manifest: bool, +} + +/// Wait for either interrupt or terminate. +/// +/// A managed worker is stopped with SIGTERM, and a process that only listens +/// for ctrl-c dies without running `shutdown_async`, which leaves its +/// Message-path triggers registered against a function that no longer exists. +#[cfg(unix)] +async fn wait_for_shutdown() -> anyhow::Result<()> { + use tokio::signal::unix::{signal, SignalKind}; + let mut terminate = signal(SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {} + } + Ok(()) +} + +#[cfg(not(unix))] +async fn wait_for_shutdown() -> anyhow::Result<()> { + tokio::signal::ctrl_c().await?; + Ok(()) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + + let cli = Cli::parse(); + + if cli.manifest { + println!( + "{}", + serde_json::to_string_pretty(&manifest::build_manifest())? + ); + return Ok(()); + } + + let iii = register_worker( + &cli.url, + InitOptions { + metadata: Some(WorkerMetadata { + runtime: "rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + name: "document".to_string(), + os: std::env::consts::OS.to_string(), + pid: Some(std::process::id()), + telemetry: None, + ..WorkerMetadata::default() + }), + ..InitOptions::default() + }, + ); + let iii = Arc::new(iii); + + // A malformed seed warns and falls through: the stored value or the + // built-in default still applies, and refusing to boot over a seed file + // would be worse than ignoring it. + let seed = cli + .config + .as_deref() + .and_then(|path| match WorkerConfig::from_file(path) { + Ok(cfg) => Some(cfg), + Err(e) => { + tracing::warn!(error = %e, path, "failed to parse config seed; ignoring it"); + None + } + }); + + configuration::register_config(&iii, seed.as_ref()) + .await + .map_err(|e| anyhow::anyhow!("configuration::register failed: {e}"))?; + let cfg = configuration::fetch_config(&iii) + .await + .map_err(|e| anyhow::anyhow!("configuration::get failed: {e}"))?; + tracing::info!( + max_input_bytes = cfg.max_input_bytes, + max_chars = cfg.max_chars, + max_assets = cfg.max_assets, + "configuration loaded" + ); + let cell: ConfigCell = Arc::new(RwLock::new(Arc::new(cfg))); + + functions::register_all(&iii, &cell, Arc::new(EngineBus::new(iii.clone()))); + + configuration::register_config_trigger(&iii, cell.clone()) + .map_err(|e| anyhow::anyhow!("configuration trigger registration failed: {e}"))?; + + tracing::info!(url = %cli.url, "document worker ready"); + + wait_for_shutdown().await?; + iii.shutdown_async().await; + Ok(()) +} diff --git a/document/src/manifest.rs b/document/src/manifest.rs new file mode 100644 index 000000000..ddc0d447a --- /dev/null +++ b/document/src/manifest.rs @@ -0,0 +1,66 @@ +//! The `--manifest` payload the registry publish pipeline reads. +//! +//! Printed without connecting to the engine, so it stays fast and +//! side-effect-free. + +use serde::Serialize; + +use crate::config::WorkerConfig; + +#[derive(Debug, Serialize)] +pub struct ModuleManifest { + pub name: String, + pub version: String, + pub description: String, + pub default_config: serde_json::Value, + pub supported_targets: Vec, +} + +pub const DESCRIPTION: &str = + "Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV and PDF documents to markdown \ + on this machine: detect the format from the bytes, convert with structure intact, pull out \ + the images embedded in them, and transcribe a scan by rendering its pages and reading them \ + with a vision model."; + +pub fn build_manifest() -> ModuleManifest { + ModuleManifest { + name: env!("CARGO_PKG_NAME").to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + description: DESCRIPTION.to_string(), + default_config: WorkerConfig::default().to_json(), + supported_targets: vec![env!("TARGET").to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `POST /publish` rejects a manifest missing any of the five fields. + #[test] + fn manifest_carries_every_required_field() { + let json = serde_json::to_value(build_manifest()).expect("manifest serializes"); + assert_eq!(json["name"], "document"); + assert!(json["version"].as_str().is_some_and(|v| !v.is_empty())); + assert!(json["description"].as_str().is_some_and(|d| d.len() > 20)); + assert!(json["default_config"].is_object()); + assert!(json["supported_targets"] + .as_array() + .is_some_and(|t| !t.is_empty())); + } + + /// The manifest name is the folder name, the binary name, and the registry + /// key. A drift here breaks the release, not the build. + #[test] + fn manifest_name_matches_the_worker_name() { + assert_eq!(build_manifest().name, "document"); + } + + #[test] + fn default_config_mirrors_the_shipped_defaults() { + assert_eq!( + build_manifest().default_config, + WorkerConfig::default().to_json() + ); + } +} diff --git a/document/src/source.rs b/document/src/source.rs new file mode 100644 index 000000000..d3e582640 --- /dev/null +++ b/document/src/source.rs @@ -0,0 +1,509 @@ +//! How a document reaches a handler, and the conventions every handler shares. +//! +//! Two shapes, one of them required: a filesystem `path`, or `bytes_base64` +//! for a document that only exists in memory — an attachment in a chat +//! composer never touches the disk. Both land as one owned buffer, because the +//! converter wants a slice and every function here reads the whole file. +//! +//! Inline bytes carry a third field that a path does not need: `file_name`. +//! CSV has no signature of its own, so without a name a spreadsheet export is +//! unrecognisable, and the converter would refuse a file it can read perfectly +//! well. + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +use crate::config::WorkerConfig; + +/// The filesystem jail a call runs under. +/// +/// The harness stamps this onto every function it dispatches, so a `path` an +/// agent supplies has to be checked against it. Without the check these +/// functions would read any document on the machine and hand back its text, +/// which is a way around the scope the session was granted. Mirrors the shape +/// the shell and pdf workers take. +#[derive(Debug, Clone, Default, Deserialize, JsonSchema)] +pub struct FsScope { + /// The session's working directory. + pub root: String, + /// Additional directories or files explicitly granted to this session. + #[serde(default)] + pub grants: Vec, +} + +/// Where the document comes from. Exactly one of `path` and `bytes_base64` +/// must be set. +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct DocumentSource { + /// Filesystem path to the document. Mutually exclusive with + /// `bytes_base64`. + #[serde(default)] + pub path: Option, + + /// Base64-encoded document bytes, for a document with no path — an + /// attachment held in memory. Mutually exclusive with `path`. + #[serde(default)] + pub bytes_base64: Option, + + /// Original file name for inline bytes, used only to recognise a format + /// the content cannot name. A `.csv` needs this; nothing else does. + /// Ignored when `path` is set, which carries its own name. + #[serde(default)] + pub file_name: Option, + + /// The filesystem jail this call runs under. Stamped by the harness on an + /// agent's call; absent on an operator or console call, which is already + /// user-initiated and not subject to the agent's scope. + #[serde(default)] + pub fs_scope: Option, +} + +impl DocumentSource { + /// Read the document into memory, enforcing the configured size ceiling + /// before anything is parsed. + pub fn load(&self, cfg: &WorkerConfig) -> Result, String> { + match (&self.path, &self.bytes_base64) { + (Some(_), Some(_)) => { + Err("provide either `path` or `bytes_base64`, not both".to_string()) + } + (None, None) => Err("provide a `path` or `bytes_base64`".to_string()), + (Some(path), None) => Self::read_file(path, self.fs_scope.as_ref(), cfg), + (None, Some(encoded)) => Self::decode(encoded, cfg), + } + } + + /// A short label for logs and responses: the file name, or a note that the + /// document arrived inline. Never the full path, which may be sensitive. + pub fn label(&self) -> String { + match self.file_name_hint() { + Some(name) => name, + None => "".to_string(), + } + } + + /// The name format detection may fall back on: the path's file name, or + /// the `file_name` supplied alongside inline bytes. + pub fn file_name_hint(&self) -> Option { + if let Some(path) = &self.path { + return Path::new(path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .or_else(|| Some(path.clone())); + } + self.file_name.clone() + } + + fn read_file( + path: &str, + scope: Option<&FsScope>, + cfg: &WorkerConfig, + ) -> Result, String> { + use std::io::Read as _; + + // Resolve before checking. A path is only inside the jail once symlinks + // and `..` are gone, and a check that follows a symlink checks the wrong + // file. + let resolved = std::fs::canonicalize(path).map_err(|e| format!("{path}: {e}"))?; + if let Some(scope) = scope { + authorize(&resolved, scope)?; + } + + // Everything after this point works on ONE open handle. Checking the + // path, then checking it again for size, then opening it a third time to + // read leaves two windows: a file swapped for a symlink between the + // authorization and the read discloses a file outside the scope, and one + // that grows between the size check and the read walks past + // `max_input_bytes`. The handle is the same file for all three. + let file = std::fs::File::open(&resolved).map_err(|e| format!("{path}: {e}"))?; + let meta = file.metadata().map_err(|e| format!("{path}: {e}"))?; + if !meta.is_file() { + return Err(format!("{path}: not a file")); + } + check_size(meta.len(), cfg)?; + + // Bounded regardless of what the metadata claimed: `take` is what makes + // the ceiling hold for a file being appended to right now, and for the + // special files whose reported length is a fiction. + let ceiling = if cfg.max_input_bytes > 0 { + cfg.max_input_bytes + } else { + u64::MAX + }; + let mut bytes = Vec::with_capacity(meta.len().min(1 << 20) as usize); + let read = file + .take(ceiling.saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(|e| format!("{path}: {e}"))?; + check_size(read as u64, cfg)?; + Ok(bytes) + } + + fn decode(encoded: &str, cfg: &WorkerConfig) -> Result, String> { + // Reject on the encoded length first: decoding a huge blob to find out + // it is too large defeats the ceiling. + check_size((encoded.len() as u64 / 4) * 3, cfg)?; + let bytes = BASE64 + .decode(encoded.as_bytes()) + .map_err(|e| format!("bytes_base64 is not valid base64: {e}"))?; + check_size(bytes.len() as u64, cfg)?; + Ok(bytes) + } +} + +/// Reject a resolved path that sits outside the session's jail. +/// +/// The comparison is on canonical paths and whole path components, so a +/// sibling directory whose name merely starts with the root (`/w/project-old` +/// against a root of `/w/project`) is not treated as inside it. +fn authorize(resolved: &Path, scope: &FsScope) -> Result<(), String> { + let allowed = std::iter::once(&scope.root).chain(scope.grants.iter()); + for entry in allowed { + // A grant that does not resolve is a stale grant, not a reason to fail + // the call: skip it and let the remaining ones decide. + let Ok(base) = std::fs::canonicalize(entry) else { + continue; + }; + if resolved == base || resolved.starts_with(&base) { + return Ok(()); + } + } + Err(format!( + "{} is outside this session's filesystem scope", + resolved.display() + )) +} + +fn check_size(bytes: u64, cfg: &WorkerConfig) -> Result<(), String> { + if cfg.max_input_bytes > 0 && bytes > cfg.max_input_bytes { + return Err(format!( + "document is {bytes} bytes, over the configured max_input_bytes of {}", + cfg.max_input_bytes + )); + } + Ok(()) +} + +/// A body that may have been shortened to fit one response, and the numbers a +/// caller needs to decide what to do about it. +/// +/// The cap is what keeps a long document from flooding a model's context. A +/// caller that genuinely wants the whole thing asks for `max_chars: 0`, which +/// is the shape a worker-to-worker pipeline uses to move a document without it +/// passing through anyone's context. Same shape the pdf worker returns, so a +/// caller handling both reads one field set. +#[derive(Debug, Serialize, JsonSchema)] +pub struct Body { + /// The markdown, shortened to the effective character cap. + pub text: String, + + /// Characters returned in `text`. + pub chars: usize, + + /// Characters the document actually holds. Equal to `chars` when nothing + /// was dropped. + pub total_chars: usize, + + /// `true` when `text` stops short of the document. Ask again with + /// `max_chars: 0` to take everything. + pub truncated: bool, + + /// Leading characters of the content. Present only when the body was + /// truncated, so a caller can see the shape of what it did not get without + /// re-reading the start of `text`. + #[serde(skip_serializing_if = "Option::is_none")] + pub preview: Option, +} + +impl Body { + /// Build a response body, applying `max_chars` (`0` means uncapped) on a + /// character boundary. + pub fn new(full: String, max_chars: usize, preview_chars: usize) -> Self { + let total_chars = full.chars().count(); + if max_chars == 0 || total_chars <= max_chars { + return Self { + chars: total_chars, + total_chars, + text: full, + truncated: false, + preview: None, + }; + } + let text: String = full.chars().take(max_chars).collect(); + let preview: String = full.chars().take(preview_chars).collect(); + Self { + chars: text.chars().count(), + total_chars, + text, + truncated: true, + preview: Some(preview), + } + } +} + +/// Turn a converter error into something the caller can act on. +/// +/// The converter's variants each imply a different next move, and its own +/// `Display` text does not say what that move is. An agent that reads +/// "unsupported input" with no advice tends to retry the same call. +pub fn describe_error(what: &str, err: &anydoc::ConvertError) -> String { + let advice = match err { + anydoc::ConvertError::Encrypted => { + "the document is encrypted; nothing here can open it, so ask for an unlocked copy" + } + anydoc::ConvertError::Unsupported(_) => { + "this format cannot be converted; for a scanned PDF call pdf::classify, which reports \ + which pages need OCR" + } + anydoc::ConvertError::Malformed { .. } => { + "the file is structurally unusable; it is likely truncated or not the format it claims" + } + anydoc::ConvertError::ResourceLimit { .. } => { + "the document crossed a fixed safety limit while parsing; it cannot be read here" + } + anydoc::ConvertError::MissingPart { .. } => { + "a part the format requires is absent; the file is incomplete" + } + anydoc::ConvertError::Io(_) => "the file could not be read", + // The converter marks its error enum non-exhaustive, so a new variant + // must not stop this from compiling. Nothing useful to advise yet. + _ => "the document could not be converted", + }; + format!("{what} failed: {err} — {advice}") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg() -> WorkerConfig { + WorkerConfig::default() + } + + #[test] + fn requires_exactly_one_input() { + let err = DocumentSource::default().load(&cfg()).expect_err("neither"); + assert!(err.contains("provide a `path`"), "{err}"); + + let both = DocumentSource { + path: Some("a.docx".into()), + bytes_base64: Some("AAAA".into()), + ..DocumentSource::default() + }; + let err = both.load(&cfg()).expect_err("both"); + assert!(err.contains("not both"), "{err}"); + } + + #[test] + fn decodes_inline_bytes() { + let src = DocumentSource { + bytes_base64: Some(BASE64.encode(b"a,b\n1,2\n")), + ..DocumentSource::default() + }; + assert_eq!(src.load(&cfg()).expect("decodes"), b"a,b\n1,2\n"); + } + + #[test] + fn rejects_malformed_base64() { + let src = DocumentSource { + bytes_base64: Some("not base64!!!".into()), + ..DocumentSource::default() + }; + let err = src.load(&cfg()).expect_err("malformed"); + assert!(err.contains("not valid base64"), "{err}"); + } + + /// The ceiling has to hold against the file itself, not only against what + /// its metadata claimed a moment earlier. + #[test] + fn a_file_over_the_ceiling_is_refused_on_the_bytes_read() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("big.csv"); + std::fs::write(&path, vec![b'a'; 4096]).expect("write"); + + let cfg = WorkerConfig { + max_input_bytes: 128, + ..WorkerConfig::default() + }; + let source = DocumentSource { + path: Some(path.to_string_lossy().to_string()), + ..DocumentSource::default() + }; + let err = source.load(&cfg).expect_err("over the ceiling"); + assert!(err.contains("max_input_bytes"), "{err}"); + } + + #[test] + fn a_file_inside_the_ceiling_reads_whole() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("rows.csv"); + std::fs::write(&path, b"a,b\n1,2\n").expect("write"); + + let source = DocumentSource { + path: Some(path.to_string_lossy().to_string()), + ..DocumentSource::default() + }; + assert_eq!(source.load(&cfg()).expect("reads"), b"a,b\n1,2\n"); + } + + #[test] + fn enforces_the_size_ceiling_before_decoding() { + let cfg = WorkerConfig { + max_input_bytes: 4, + ..WorkerConfig::default() + }; + let src = DocumentSource { + bytes_base64: Some(BASE64.encode(vec![0u8; 1024])), + ..DocumentSource::default() + }; + let err = src.load(&cfg).expect_err("over the ceiling"); + assert!(err.contains("max_input_bytes"), "{err}"); + } + + /// The harness stamps a scope on every call it dispatches. Without this + /// check an agent could read any document on the machine and get its text + /// back, which is a way around the scope its session was granted. + #[test] + fn a_path_outside_the_session_scope_is_refused() { + let dir = tempfile::tempdir().expect("temp dir"); + let inside = dir.path().join("report.docx"); + std::fs::write(&inside, b"PK\x03\x04").expect("write"); + + let outside = tempfile::tempdir().expect("second temp dir"); + let secret = outside.path().join("payroll.xlsx"); + std::fs::write(&secret, b"PK\x03\x04").expect("write"); + + let scope = FsScope { + root: dir.path().to_string_lossy().to_string(), + grants: vec![], + }; + + let allowed = DocumentSource { + path: Some(inside.to_string_lossy().to_string()), + fs_scope: Some(scope.clone()), + ..DocumentSource::default() + }; + assert!(allowed.load(&cfg()).is_ok(), "a path inside the root reads"); + + let refused = DocumentSource { + path: Some(secret.to_string_lossy().to_string()), + fs_scope: Some(scope), + ..DocumentSource::default() + }; + let err = refused.load(&cfg()).expect_err("outside the scope"); + assert!( + err.contains("outside this session's filesystem scope"), + "{err}" + ); + } + + /// A sibling whose name merely starts with the root is not inside it. A + /// prefix comparison on strings would let `/w/project-old` pass for a root + /// of `/w/project`. + #[test] + fn a_sibling_directory_with_a_shared_prefix_is_not_inside_the_scope() { + let parent = tempfile::tempdir().expect("temp dir"); + let root = parent.path().join("project"); + let sibling = parent.path().join("project-old"); + std::fs::create_dir_all(&root).expect("root"); + std::fs::create_dir_all(&sibling).expect("sibling"); + let doc = sibling.join("secret.docx"); + std::fs::write(&doc, b"PK\x03\x04").expect("write"); + + let source = DocumentSource { + path: Some(doc.to_string_lossy().to_string()), + fs_scope: Some(FsScope { + root: root.to_string_lossy().to_string(), + grants: vec![], + }), + ..DocumentSource::default() + }; + let err = source.load(&cfg()).expect_err("sibling is outside"); + assert!(err.contains("outside"), "{err}"); + } + + #[test] + fn label_never_leaks_the_directory() { + let src = DocumentSource { + path: Some("/home/someone/private/report.docx".into()), + ..DocumentSource::default() + }; + assert_eq!(src.label(), "report.docx"); + assert_eq!(DocumentSource::default().label(), ""); + } + + /// Inline bytes are the composer's path, and the name is the only way a + /// CSV is ever recognised. + #[test] + fn inline_bytes_keep_their_name_for_detection() { + let src = DocumentSource { + bytes_base64: Some(BASE64.encode(b"a,b\n")), + file_name: Some("rows.csv".into()), + ..DocumentSource::default() + }; + assert_eq!(src.file_name_hint().as_deref(), Some("rows.csv")); + assert_eq!(src.label(), "rows.csv"); + } + + #[test] + fn body_reports_what_it_dropped() { + let body = Body::new("abcdefghij".to_string(), 4, 2); + assert_eq!(body.text, "abcd"); + assert_eq!(body.chars, 4); + assert_eq!(body.total_chars, 10); + assert!(body.truncated); + assert_eq!(body.preview.as_deref(), Some("ab")); + } + + #[test] + fn body_uncapped_when_max_chars_is_zero() { + let body = Body::new("abcdefghij".to_string(), 0, 2); + assert_eq!(body.text, "abcdefghij"); + assert!(!body.truncated); + assert!(body.preview.is_none()); + } + + /// Truncation must not split a multi-byte character. + #[test] + fn body_truncates_on_character_boundaries() { + let body = Body::new("日本語のテキスト".to_string(), 3, 2); + assert_eq!(body.text, "日本語"); + assert_eq!(body.total_chars, 8); + } + + /// Each failure implies a different next move, and the converter's own + /// message never says what it is. + #[test] + fn errors_say_what_to_do_next() { + let encrypted = describe_error("conversion", &anydoc::ConvertError::Encrypted); + assert!(encrypted.contains("unlocked copy"), "{encrypted}"); + + let unsupported = + describe_error("conversion", &anydoc::ConvertError::Unsupported("x".into())); + assert!(unsupported.contains("pdf::classify"), "{unsupported}"); + + let malformed = describe_error( + "conversion", + &anydoc::ConvertError::Malformed { + part: Some("word/document.xml".into()), + detail: "unexpected end".into(), + }, + ); + assert!(malformed.contains("truncated"), "{malformed}"); + } + + #[test] + fn a_new_converter_variant_still_gets_advice() { + // The converter's error enum is `#[non_exhaustive]`, so the catch-all + // arm is what a future variant lands on. It still has to name the + // document and read as an answer. + let described = describe_error( + "conversion", + &anydoc::ConvertError::Io(std::io::Error::other("disk went away")), + ); + assert!(described.contains("conversion failed"), "{described}"); + assert!(described.contains("could not be read"), "{described}"); + } +} diff --git a/document/tests/fixtures/README.md b/document/tests/fixtures/README.md new file mode 100644 index 000000000..84d65670f --- /dev/null +++ b/document/tests/fixtures/README.md @@ -0,0 +1,23 @@ +# Test fixtures + +Hand-built documents, generated by `make_fixtures.py` in this directory. They +are assembled from the parts each format requires rather than exported from an +office suite, so they stay small, their content is known exactly, and they carry +no third-party licensing. + +| File | What it exercises | +|---|---| +| `sample.docx` | Prose with a heading, a paragraph and a two-row table. A table that arrives as a run-on paragraph is the regression this catches. | +| `sample.xlsx` | A workbook with inline strings and no shared string table — the shape a machine-generated export takes. | +| `sample.pptx` | A one-slide deck carrying an embedded PNG, so asset extraction has real bytes to return. | +| `sample.rtf` | A signature-carrying format that is not a ZIP package. | +| `sample.csv` | The one format with no signature at all: recognised only by its name. | + +Regenerate with: + +```bash +python3 tests/fixtures/make_fixtures.py +``` + +The generated files are committed. A converter upgrade that changes what these +documents produce should show up as a failing assertion, which is the point. diff --git a/document/tests/fixtures/make_fixtures.py b/document/tests/fixtures/make_fixtures.py new file mode 100644 index 000000000..13a69d117 --- /dev/null +++ b/document/tests/fixtures/make_fixtures.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Regenerate the committed fixture corpus in this directory. + +The files are assembled here from the parts each format requires rather than +exported from an office suite, so they stay under a few kilobytes, their +content is known exactly, and they carry no third-party licensing. + +Usage: + + python3 tests/fixtures/make_fixtures.py +""" + +import base64 +import zipfile +from pathlib import Path + +OUT = Path(__file__).resolve().parent + +# Fixed timestamp so a regeneration with no content change produces a +# byte-identical file and shows up as no diff at all. +ZIP_DATE = (2026, 1, 1, 0, 0, 0) + +# A 1x1 red PNG. Small enough to read as a literal, real enough that a decoder +# accepts it, which is what the asset assertions need. +DOT_PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8" + "z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + +RELS_NS = 'xmlns="http://schemas.openxmlformats.org/package/2006/relationships"' +CT_NS = 'xmlns="http://schemas.openxmlformats.org/package/2006/content-types"' +REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + + +def write_zip(path: Path, entries): + """Write a package with deterministic entry order and timestamps.""" + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in entries: + info = zipfile.ZipInfo(name, date_time=ZIP_DATE) + info.compress_type = zipfile.ZIP_DEFLATED + zf.writestr(info, data) + print(f"wrote {path.name} ({path.stat().st_size} bytes)") + + +def docx(): + """A Word document: one heading, one paragraph, one two-row table.""" + ct = f""" + + + + +""" + + rels = f""" + + +""" + + w = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' + document = f""" + +Quarterly Notes +The engine handled every request without a restart. + +MetricValue +Requests21480 + +""" + + write_zip( + OUT / "sample.docx", + [ + ("[Content_Types].xml", ct), + ("_rels/.rels", rels), + ("word/document.xml", document), + ], + ) + + +def xlsx(): + """A workbook: one sheet, inline strings, no shared string table.""" + ct = f""" + + + + + +""" + + rels = f""" + + +""" + + ns = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"' + r_ns = 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"' + workbook = f""" + + +""" + + wb_rels = f""" + + +""" + + sheet = f""" + +scenariojobs per second +echo21480 +fanout2184 +""" + + write_zip( + OUT / "sample.xlsx", + [ + ("[Content_Types].xml", ct), + ("_rels/.rels", rels), + ("xl/workbook.xml", workbook), + ("xl/_rels/workbook.xml.rels", wb_rels), + ("xl/worksheets/sheet1.xml", sheet), + ], + ) + + +def pptx(): + """A one-slide deck carrying an embedded image, so asset extraction has + something real to pull out.""" + ct = f""" + + + + + + +""" + + rels = f""" + + +""" + + p_ns = ( + 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" ' + 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" ' + 'xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"' + ) + + presentation = f""" + + +""" + + pres_rels = f""" + + +""" + + slide = f""" + + + + + + +Three Primitives + + + + +Worker, function, trigger. + + + + + + +""" + + slide_rels = f""" + + +""" + + write_zip( + OUT / "sample.pptx", + [ + ("[Content_Types].xml", ct), + ("_rels/.rels", rels), + ("ppt/presentation.xml", presentation), + ("ppt/_rels/presentation.xml.rels", pres_rels), + ("ppt/slides/slide1.xml", slide), + ("ppt/slides/_rels/slide1.xml.rels", slide_rels), + ("ppt/media/image1.png", DOT_PNG), + ], + ) + + +def rtf(): + """Rich Text is plain text on the wire, which makes it the cheapest check + that a signature-carrying non-package format is recognised.""" + path = OUT / "sample.rtf" + path.write_text( + r"{\rtf1\ansi\deff0 {\b Release notes}\par The queue drained in 40 ms.\par}", + encoding="ascii", + ) + print(f"wrote {path.name} ({path.stat().st_size} bytes)") + + +def csv(): + path = OUT / "sample.csv" + path.write_text("scenario,jobs_per_second\necho,21480\nfanout,2184\n", encoding="utf-8") + print(f"wrote {path.name} ({path.stat().st_size} bytes)") + + +if __name__ == "__main__": + docx() + xlsx() + pptx() + rtf() + csv() diff --git a/document/tests/fixtures/sample.csv b/document/tests/fixtures/sample.csv new file mode 100644 index 000000000..87b423cbe --- /dev/null +++ b/document/tests/fixtures/sample.csv @@ -0,0 +1,3 @@ +scenario,jobs_per_second +echo,21480 +fanout,2184 diff --git a/document/tests/fixtures/sample.docx b/document/tests/fixtures/sample.docx new file mode 100644 index 000000000..87b427bd2 Binary files /dev/null and b/document/tests/fixtures/sample.docx differ diff --git a/document/tests/fixtures/sample.pptx b/document/tests/fixtures/sample.pptx new file mode 100644 index 000000000..9dfb7eb30 Binary files /dev/null and b/document/tests/fixtures/sample.pptx differ diff --git a/document/tests/fixtures/sample.rtf b/document/tests/fixtures/sample.rtf new file mode 100644 index 000000000..09342cfe0 --- /dev/null +++ b/document/tests/fixtures/sample.rtf @@ -0,0 +1 @@ +{\rtf1\ansi\deff0 {\b Release notes}\par The queue drained in 40 ms.\par} \ No newline at end of file diff --git a/document/tests/fixtures/sample.xlsx b/document/tests/fixtures/sample.xlsx new file mode 100644 index 000000000..4decb4bf4 Binary files /dev/null and b/document/tests/fixtures/sample.xlsx differ diff --git a/document/tests/formats.rs b/document/tests/formats.rs new file mode 100644 index 000000000..daa8a0c5d --- /dev/null +++ b/document/tests/formats.rs @@ -0,0 +1,245 @@ +//! Every format this worker claims, converted end to end through the handler. +//! +//! The unit tests cover the routing and the caps with CSV, which needs no +//! binary fixture. These cover the claim on the box: a Word file, a workbook, a +//! deck and an RTF document all come out as markdown, and a deck's embedded +//! image comes back as bytes a model can be handed. +//! +//! The fixtures are assembled by `tests/fixtures/make_fixtures.py` from the +//! parts each format requires, so a converter upgrade that changes what they +//! produce shows up as a failing assertion here, which is the point. + +use std::path::PathBuf; + +use document::config::WorkerConfig; +use document::format::{DetectedFrom, Family, Format}; +use document::functions::{assets, detect, markdown}; +use document::source::DocumentSource; + +fn fixture(name: &str) -> String { + let path: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(name); + path.to_string_lossy().to_string() +} + +fn source(name: &str) -> DocumentSource { + DocumentSource { + path: Some(fixture(name)), + ..DocumentSource::default() + } +} + +fn convert(name: &str) -> markdown::Response { + markdown::handle( + markdown::Request { + source: source(name), + format: None, + max_chars: Some(0), + }, + &WorkerConfig::default(), + ) + .unwrap_or_else(|e| panic!("{name} converts: {e}")) +} + +fn detect(name: &str) -> detect::Response { + detect::handle( + detect::Request { + source: source(name), + }, + &WorkerConfig::default(), + ) + .unwrap_or_else(|e| panic!("{name} is detected: {e}")) +} + +#[test] +fn a_word_document_keeps_its_headings_and_tables() { + let response = convert("sample.docx"); + assert_eq!(response.format, Format::Docx); + assert_eq!(response.family, Family::Prose); + assert_eq!(response.detected_from, DetectedFrom::Content); + + let text = &response.body.text; + assert!(text.contains("Quarterly Notes"), "{text}"); + assert!( + text.contains("without a restart"), + "body text survived: {text}" + ); + // A table that arrives as a run-on paragraph is the failure this asserts + // against: the pipe is what makes it a table to the model. + assert!(text.contains('|'), "the table became prose: {text}"); + assert!(text.contains("21480"), "{text}"); + assert!(!response.body.truncated); +} + +#[test] +fn a_workbook_becomes_rows_a_model_can_read() { + let response = convert("sample.xlsx"); + assert_eq!(response.format, Format::Excel); + assert_eq!(response.family, Family::Spreadsheet); + + let text = &response.body.text; + assert!(text.contains("scenario"), "{text}"); + assert!(text.contains("echo"), "{text}"); + assert!(text.contains("21480"), "{text}"); +} + +#[test] +fn a_deck_converts_and_reports_the_images_markdown_dropped() { + let response = convert("sample.pptx"); + assert_eq!(response.format, Format::Pptx); + assert_eq!(response.family, Family::Presentation); + + let text = &response.body.text; + assert!(text.contains("Three Primitives"), "{text}"); + assert!(text.contains("Worker, function, trigger."), "{text}"); + + // The count is the whole reason it is on the response: markdown renders an + // embedded image as alt text, so without this a deck of diagrams looks + // like a document that simply had little to say. + assert_eq!(response.asset_count, 1); +} + +#[test] +fn an_rtf_document_converts() { + let response = convert("sample.rtf"); + assert_eq!(response.format, Format::Rtf); + assert_eq!(response.family, Family::Prose); + assert!(response.body.text.contains("Release notes")); + assert!(response.body.text.contains("40 ms")); +} + +#[test] +fn a_csv_file_on_disk_is_recognised_by_its_name() { + let response = convert("sample.csv"); + assert_eq!(response.format, Format::Csv); + assert_eq!(response.detected_from, DetectedFrom::Extension); + assert!(response.body.text.contains("fanout")); +} + +/// The images are the point of the extraction: a deck whose content is +/// diagrams reads as empty without them, and a model that can see images can +/// use the bytes directly. +#[test] +fn a_decks_image_comes_back_as_usable_bytes() { + let response = assets::handle( + assets::Request { + source: source("sample.pptx"), + format: None, + max_assets: None, + media_type_prefix: Some("image/".to_string()), + include_bytes: true, + }, + &WorkerConfig::default(), + ) + .expect("the deck's assets extract"); + + assert_eq!(response.total_count, 1); + assert!(!response.truncated); + let asset = &response.assets[0]; + assert_eq!(asset.media_type, "image/png"); + assert!(asset.omitted.is_none()); + + let encoded = asset.bytes_base64.as_ref().expect("bytes were included"); + let decoded = base64_decode(encoded); + assert_eq!( + &decoded[..8], + b"\x89PNG\r\n\x1a\n", + "what came back is not a PNG" + ); + assert_eq!(asset.size_bytes as usize, decoded.len()); +} + +/// An inventory pass is how a caller decides whether the bytes are worth +/// moving at all, so it must still report the type and the size. +#[test] +fn an_inventory_lists_assets_without_moving_them() { + let response = assets::handle( + assets::Request { + source: source("sample.pptx"), + format: None, + max_assets: None, + media_type_prefix: None, + include_bytes: false, + }, + &WorkerConfig::default(), + ) + .expect("the deck's assets are listed"); + + let asset = &response.assets[0]; + assert!(asset.bytes_base64.is_none()); + assert_eq!(asset.omitted, Some("not_requested")); + assert!(asset.size_bytes > 0); +} + +/// An asset over the per-asset ceiling is still announced. Dropping it from +/// the list entirely would tell the caller the document has no images. +#[test] +fn an_oversized_asset_is_listed_without_its_bytes() { + let cfg = WorkerConfig { + max_asset_bytes: 4, + ..WorkerConfig::default() + }; + let response = assets::handle( + assets::Request { + source: source("sample.pptx"), + format: None, + max_assets: None, + media_type_prefix: None, + include_bytes: true, + }, + &cfg, + ) + .expect("extraction succeeds"); + + let asset = &response.assets[0]; + assert!(asset.bytes_base64.is_none()); + assert_eq!(asset.omitted, Some("too_large")); + assert_eq!(asset.media_type, "image/png"); +} + +/// The per-asset ceiling does not bound a response on its own: a couple of dozen +/// assets each just under it still add up to a payload nobody asked for. The +/// total budget stops the encoding while still listing what exists. +#[test] +fn the_response_budget_stops_encoding_but_not_listing() { + let cfg = WorkerConfig { + max_assets_total_bytes: 1, + ..WorkerConfig::default() + }; + let response = assets::handle( + assets::Request { + source: source("sample.pptx"), + format: None, + max_assets: None, + media_type_prefix: None, + include_bytes: true, + }, + &cfg, + ) + .expect("extraction succeeds"); + + let asset = &response.assets[0]; + assert!(asset.bytes_base64.is_none()); + assert_eq!(asset.omitted, Some("budget_spent")); + // Still announced, with everything a caller needs to fetch it alone. + assert_eq!(asset.media_type, "image/png"); + assert!(asset.size_bytes > 0); +} + +/// Detection runs on the bytes, so a package format is recognised without its +/// name — which is the case for a file pasted into a composer. +#[test] +fn detection_reads_the_package_not_the_extension() { + let response = detect("sample.pptx"); + assert_eq!(response.format, Some(Format::Pptx)); + assert_eq!(response.detected_from, Some(DetectedFrom::Content)); + assert!(response.convertible); + assert!(response.has_assets); +} + +fn base64_decode(encoded: &str) -> Vec { + use base64::engine::general_purpose::STANDARD; + use base64::Engine as _; + STANDARD.decode(encoded).expect("valid base64") +} diff --git a/document/tests/golden/schemas/document.detect.json b/document/tests/golden/schemas/document.detect.json new file mode 100644 index 000000000..b2ff4e936 --- /dev/null +++ b/document/tests/golden/schemas/document.detect.json @@ -0,0 +1,298 @@ +{ + "description": "Identify a document's format from its bytes (falling back to the file name for CSV, which carries no signature), and report which family it belongs to and whether this worker can convert it. Microseconds, and no conversion.", + "function_id": "document::detect", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell and pdf workers take.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + } + }, + "description": "Where the document comes from. Exactly one of `path` and `bytes_base64` must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded document bytes, for a document with no path — an attachment held in memory. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "file_name": { + "default": null, + "description": "Original file name for inline bytes, used only to recognise a format the content cannot name. A `.csv` needs this; nothing else does. Ignored when `path` is set, which carries its own name.", + "type": [ + "string", + "null" + ] + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "path": { + "default": null, + "description": "Filesystem path to the document. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "DetectedFrom": { + "description": "How the format was arrived at, weakest claim last.", + "oneOf": [ + { + "description": "The caller named it, and the bytes were not consulted.", + "enum": [ + "requested" + ], + "type": "string" + }, + { + "description": "The signature the format's specification designates (PDF header, RTF open group, OLE stream names, ZIP package mimetype).", + "enum": [ + "content" + ], + "type": "string" + }, + { + "description": "The file extension only. CSV carries no signature, so this is the only way it is ever recognised; for any other format it means the content did not match anything known.", + "enum": [ + "extension" + ], + "type": "string" + } + ] + }, + "Family": { + "description": "What the document is, rather than which program wrote it.\n\nA caller routing a mixed bag of attachments cares that a file is a spreadsheet, not that it is `.ods` rather than `.xlsx`.", + "oneOf": [ + { + "description": "Prose: Word, OpenDocument Text, RTF.", + "enum": [ + "prose" + ], + "type": "string" + }, + { + "description": "Rows and columns: Excel, OpenDocument Spreadsheet, CSV.", + "enum": [ + "spreadsheet" + ], + "type": "string" + }, + { + "description": "Slides: PowerPoint, OpenDocument Presentation.", + "enum": [ + "presentation" + ], + "type": "string" + }, + { + "description": "A book: EPUB.", + "enum": [ + "book" + ], + "type": "string" + }, + { + "description": "PDF, which is its own family because it is the one format with a dedicated worker and a page-level OCR decision.", + "enum": [ + "pdf" + ], + "type": "string" + } + ] + }, + "Format": { + "description": "A format this worker converts. The names are the wire vocabulary: stable, lowercase, and independent of the file extension that named them (`.docm` is `docx`, `.xlsb` is `excel`).", + "oneOf": [ + { + "description": "Binary Word 97-2003 (`.doc`).", + "enum": [ + "doc" + ], + "type": "string" + }, + { + "description": "WordprocessingML (`.docx`, `.docm`).", + "enum": [ + "docx" + ], + "type": "string" + }, + { + "description": "OpenDocument Text (`.odt`).", + "enum": [ + "odt" + ], + "type": "string" + }, + { + "description": "Rich Text Format (`.rtf`).", + "enum": [ + "rtf" + ], + "type": "string" + }, + { + "description": "Binary PowerPoint 97-2003 (`.ppt`, `.pps`, `.pot`).", + "enum": [ + "ppt" + ], + "type": "string" + }, + { + "description": "PresentationML (`.pptx`, `.pptm`, `.ppsx`, `.ppsm`).", + "enum": [ + "pptx" + ], + "type": "string" + }, + { + "description": "OpenDocument Presentation (`.odp`).", + "enum": [ + "odp" + ], + "type": "string" + }, + { + "description": "Excel workbooks in every container (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`).", + "enum": [ + "excel" + ], + "type": "string" + }, + { + "description": "OpenDocument Spreadsheet (`.ods`).", + "enum": [ + "ods" + ], + "type": "string" + }, + { + "description": "Delimiter-separated text (`.csv`).", + "enum": [ + "csv" + ], + "type": "string" + }, + { + "description": "EPUB 2 and 3 (`.epub`).", + "enum": [ + "epub" + ], + "type": "string" + }, + { + "description": "Portable Document Format (`.pdf`).", + "enum": [ + "pdf" + ], + "type": "string" + } + ] + } + }, + "properties": { + "convertible": { + "description": "`true` when `document::to-markdown` can convert this file.", + "type": "boolean" + }, + "detected_from": { + "anyOf": [ + { + "$ref": "#/definitions/DetectedFrom" + }, + { + "type": "null" + } + ], + "description": "How the format was arrived at. `extension` is the weaker claim: the content matched nothing known, and only the file name suggested this." + }, + "elapsed_ms": { + "description": "Wall-clock time for the detection.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "family": { + "anyOf": [ + { + "$ref": "#/definitions/Family" + }, + { + "type": "null" + } + ], + "description": "What the document is: prose, a spreadsheet, a presentation, a book, a PDF. Absent when the format is unknown." + }, + "format": { + "anyOf": [ + { + "$ref": "#/definitions/Format" + }, + { + "type": "null" + } + ], + "description": "The format, or `null` when nothing recognised it. A null means the file is not one of the formats this worker reads — an image, an archive, a plain text file — not that it is broken." + }, + "has_assets": { + "description": "`true` when the format can carry embedded assets for `document::extract-assets` to pull out. False for a PDF, which converts straight to markdown without a document model, and for a CSV, which is rows of text with nowhere to put a picture. A caller routing on this should not spend a call to be told a spreadsheet has no images.", + "type": "boolean" + }, + "size_bytes": { + "description": "Size of the document in bytes.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document that arrived without one.", + "type": "string" + } + }, + "required": [ + "convertible", + "elapsed_ms", + "has_assets", + "size_bytes", + "source" + ], + "title": "Response", + "type": "object" + } +} diff --git a/document/tests/golden/schemas/document.extract-assets.json b/document/tests/golden/schemas/document.extract-assets.json new file mode 100644 index 000000000..a5d6ca710 --- /dev/null +++ b/document/tests/golden/schemas/document.extract-assets.json @@ -0,0 +1,381 @@ +{ + "description": "Pull the images and embedded objects out of a document as base64, for a deck or report whose content is pictures rather than text. Capped per response and per asset; anything left out is still listed with its type and size. Not available for PDFs — use pdf::extract-regions.", + "function_id": "document::extract-assets", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Format": { + "description": "A format this worker converts. The names are the wire vocabulary: stable, lowercase, and independent of the file extension that named them (`.docm` is `docx`, `.xlsb` is `excel`).", + "oneOf": [ + { + "description": "Binary Word 97-2003 (`.doc`).", + "enum": [ + "doc" + ], + "type": "string" + }, + { + "description": "WordprocessingML (`.docx`, `.docm`).", + "enum": [ + "docx" + ], + "type": "string" + }, + { + "description": "OpenDocument Text (`.odt`).", + "enum": [ + "odt" + ], + "type": "string" + }, + { + "description": "Rich Text Format (`.rtf`).", + "enum": [ + "rtf" + ], + "type": "string" + }, + { + "description": "Binary PowerPoint 97-2003 (`.ppt`, `.pps`, `.pot`).", + "enum": [ + "ppt" + ], + "type": "string" + }, + { + "description": "PresentationML (`.pptx`, `.pptm`, `.ppsx`, `.ppsm`).", + "enum": [ + "pptx" + ], + "type": "string" + }, + { + "description": "OpenDocument Presentation (`.odp`).", + "enum": [ + "odp" + ], + "type": "string" + }, + { + "description": "Excel workbooks in every container (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`).", + "enum": [ + "excel" + ], + "type": "string" + }, + { + "description": "OpenDocument Spreadsheet (`.ods`).", + "enum": [ + "ods" + ], + "type": "string" + }, + { + "description": "Delimiter-separated text (`.csv`).", + "enum": [ + "csv" + ], + "type": "string" + }, + { + "description": "EPUB 2 and 3 (`.epub`).", + "enum": [ + "epub" + ], + "type": "string" + }, + { + "description": "Portable Document Format (`.pdf`).", + "enum": [ + "pdf" + ], + "type": "string" + } + ] + }, + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell and pdf workers take.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + } + }, + "description": "Where the document comes from. Exactly one of `path` and `bytes_base64` must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded document bytes, for a document with no path — an attachment held in memory. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "file_name": { + "default": null, + "description": "Original file name for inline bytes, used only to recognise a format the content cannot name. A `.csv` needs this; nothing else does. Ignored when `path` is set, which carries its own name.", + "type": [ + "string", + "null" + ] + }, + "format": { + "anyOf": [ + { + "$ref": "#/definitions/Format" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Force a format instead of detecting one." + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "include_bytes": { + "default": true, + "description": "Include the base64 payload. Set `false` to inventory a document — what it holds and how big — without moving the bytes.", + "type": "boolean" + }, + "max_assets": { + "default": null, + "description": "Assets to return in this response. Narrows the configured ceiling; it cannot raise it.", + "format": "uint", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "media_type_prefix": { + "default": null, + "description": "Return only assets whose media type starts with this, e.g. `image/`. Omit for every asset.", + "type": [ + "string", + "null" + ] + }, + "path": { + "default": null, + "description": "Filesystem path to the document. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Asset": { + "description": "One embedded asset. `bytes_base64` is absent when the caller asked for an inventory, or when this asset is over the per-asset ceiling — `omitted` says which.", + "properties": { + "bytes_base64": { + "description": "The payload, base64-encoded.", + "type": [ + "string", + "null" + ] + }, + "index": { + "description": "Position in the document's asset list, stable for a given document.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "media_type": { + "description": "MIME type, e.g. `image/png`.", + "type": "string" + }, + "omitted": { + "description": "Why the payload is absent, when it is: `not_requested`, `too_large` (this asset alone is over the per-asset ceiling), or `budget_spent` (the response's total byte budget went on earlier assets — ask for this one on its own).", + "type": [ + "string", + "null" + ] + }, + "origin_part": { + "description": "The package part or stream it came from, for provenance.", + "type": "string" + }, + "size_bytes": { + "description": "Size of the payload in bytes, whether or not the payload is included.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + } + }, + "required": [ + "index", + "media_type", + "origin_part", + "size_bytes" + ], + "type": "object" + }, + "Format": { + "description": "A format this worker converts. The names are the wire vocabulary: stable, lowercase, and independent of the file extension that named them (`.docm` is `docx`, `.xlsb` is `excel`).", + "oneOf": [ + { + "description": "Binary Word 97-2003 (`.doc`).", + "enum": [ + "doc" + ], + "type": "string" + }, + { + "description": "WordprocessingML (`.docx`, `.docm`).", + "enum": [ + "docx" + ], + "type": "string" + }, + { + "description": "OpenDocument Text (`.odt`).", + "enum": [ + "odt" + ], + "type": "string" + }, + { + "description": "Rich Text Format (`.rtf`).", + "enum": [ + "rtf" + ], + "type": "string" + }, + { + "description": "Binary PowerPoint 97-2003 (`.ppt`, `.pps`, `.pot`).", + "enum": [ + "ppt" + ], + "type": "string" + }, + { + "description": "PresentationML (`.pptx`, `.pptm`, `.ppsx`, `.ppsm`).", + "enum": [ + "pptx" + ], + "type": "string" + }, + { + "description": "OpenDocument Presentation (`.odp`).", + "enum": [ + "odp" + ], + "type": "string" + }, + { + "description": "Excel workbooks in every container (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`).", + "enum": [ + "excel" + ], + "type": "string" + }, + { + "description": "OpenDocument Spreadsheet (`.ods`).", + "enum": [ + "ods" + ], + "type": "string" + }, + { + "description": "Delimiter-separated text (`.csv`).", + "enum": [ + "csv" + ], + "type": "string" + }, + { + "description": "EPUB 2 and 3 (`.epub`).", + "enum": [ + "epub" + ], + "type": "string" + }, + { + "description": "Portable Document Format (`.pdf`).", + "enum": [ + "pdf" + ], + "type": "string" + } + ] + } + }, + "properties": { + "assets": { + "description": "The assets, in document order, up to the effective ceiling.", + "items": { + "$ref": "#/definitions/Asset" + }, + "type": "array" + }, + "elapsed_ms": { + "description": "Wall-clock time for the extraction.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "format": { + "allOf": [ + { + "$ref": "#/definitions/Format" + } + ], + "description": "The format that was parsed." + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + }, + "total_count": { + "description": "Assets the document holds after `media_type_prefix` is applied. Larger than `assets.len()` when the ceiling cut the response short.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "truncated": { + "description": "`true` when the ceiling cut the response short.", + "type": "boolean" + } + }, + "required": [ + "assets", + "elapsed_ms", + "format", + "source", + "total_count", + "truncated" + ], + "title": "Response", + "type": "object" + } +} diff --git a/document/tests/golden/schemas/document.ocr.json b/document/tests/golden/schemas/document.ocr.json new file mode 100644 index 000000000..a25b46722 --- /dev/null +++ b/document/tests/golden/schemas/document.ocr.json @@ -0,0 +1,235 @@ +{ + "description": "Transcribe a document that holds no readable text: a scanned PDF, a photographed page, or a deck whose content is pictures. Renders the pages that need it and reads them with a vision model, so it costs money per page — pass `pages` (pdf::classify names them) to narrow it. Needs the browser worker for PDFs and a vision model through llm-router.", + "function_id": "document::ocr", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell and pdf workers take.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + } + }, + "description": "Where the document comes from. Exactly one of `path` and `bytes_base64` must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded document bytes, for a document with no path — an attachment held in memory. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "file_name": { + "default": null, + "description": "Original file name for inline bytes, used only to recognise a format the content cannot name. A `.csv` needs this; nothing else does. Ignored when `path` is set, which carries its own name.", + "type": [ + "string", + "null" + ] + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "max_chars": { + "default": null, + "description": "Characters to return before truncating. Omit for the configured default; `0` returns everything transcribed.", + "format": "uint", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "model": { + "default": null, + "description": "Vision model to read with. Omit for the configured default. The model is checked for vision support before anything is rendered.", + "type": [ + "string", + "null" + ] + }, + "pages": { + "default": null, + "description": "1-indexed pages to transcribe, for a PDF. Omit for every page up to the configured ceiling. This is the cost control: `pdf::classify` reports which pages are scans, and passing that list keeps a long report from being read a page at a time when only its cover is an image.", + "items": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "type": [ + "array", + "null" + ] + }, + "path": { + "default": null, + "description": "Filesystem path to the document. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Body": { + "description": "A body that may have been shortened to fit one response, and the numbers a caller needs to decide what to do about it.\n\nThe cap is what keeps a long document from flooding a model's context. A caller that genuinely wants the whole thing asks for `max_chars: 0`, which is the shape a worker-to-worker pipeline uses to move a document without it passing through anyone's context. Same shape the pdf worker returns, so a caller handling both reads one field set.", + "properties": { + "chars": { + "description": "Characters returned in `text`.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "preview": { + "description": "Leading characters of the content. Present only when the body was truncated, so a caller can see the shape of what it did not get without re-reading the start of `text`.", + "type": [ + "string", + "null" + ] + }, + "text": { + "description": "The markdown, shortened to the effective character cap.", + "type": "string" + }, + "total_chars": { + "description": "Characters the document actually holds. Equal to `chars` when nothing was dropped.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "truncated": { + "description": "`true` when `text` stops short of the document. Ask again with `max_chars: 0` to take everything.", + "type": "boolean" + } + }, + "required": [ + "chars", + "text", + "total_chars", + "truncated" + ], + "type": "object" + }, + "PageText": { + "description": "What one page turned into.", + "properties": { + "cached": { + "description": "`true` when this page came from the cache rather than the model.", + "type": "boolean" + }, + "chars": { + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "page": { + "description": "1-indexed page number, or the asset's index for an office document.", + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "text": { + "description": "The transcription. Empty when the page held no legible text.", + "type": "string" + } + }, + "required": [ + "cached", + "chars", + "page", + "text" + ], + "type": "object" + } + }, + "properties": { + "body": { + "allOf": [ + { + "$ref": "#/definitions/Body" + } + ], + "description": "The joined transcription, capped per `max_chars`." + }, + "elapsed_ms": { + "description": "Wall-clock time, rendering included.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "model": { + "description": "The model that read them.", + "type": "string" + }, + "pages": { + "description": "Per-page transcriptions, in order.", + "items": { + "$ref": "#/definitions/PageText" + }, + "type": "array" + }, + "pages_cached": { + "description": "Pages served from the cache, costing nothing.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "pages_transcribed": { + "description": "Pages actually read by the model this call. Excludes cache hits, so this is what was paid for.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + }, + "via": { + "description": "How the pixels were obtained: `image`, `pdf-render` or `document-assets`.", + "type": "string" + } + }, + "required": [ + "body", + "elapsed_ms", + "model", + "pages", + "pages_cached", + "pages_transcribed", + "source", + "via" + ], + "title": "Response", + "type": "object" + } +} diff --git a/document/tests/golden/schemas/document.to-markdown.json b/document/tests/golden/schemas/document.to-markdown.json new file mode 100644 index 000000000..33f964b61 --- /dev/null +++ b/document/tests/golden/schemas/document.to-markdown.json @@ -0,0 +1,441 @@ +{ + "description": "Convert a Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV or PDF document to markdown, preserving headings, lists, links and tables. The format is detected from the bytes. Responses are capped; pass max_chars 0 to take the whole document. For a PDF prefer pdf::classify first, which reports which pages need OCR.", + "function_id": "document::to-markdown", + "request_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Format": { + "description": "A format this worker converts. The names are the wire vocabulary: stable, lowercase, and independent of the file extension that named them (`.docm` is `docx`, `.xlsb` is `excel`).", + "oneOf": [ + { + "description": "Binary Word 97-2003 (`.doc`).", + "enum": [ + "doc" + ], + "type": "string" + }, + { + "description": "WordprocessingML (`.docx`, `.docm`).", + "enum": [ + "docx" + ], + "type": "string" + }, + { + "description": "OpenDocument Text (`.odt`).", + "enum": [ + "odt" + ], + "type": "string" + }, + { + "description": "Rich Text Format (`.rtf`).", + "enum": [ + "rtf" + ], + "type": "string" + }, + { + "description": "Binary PowerPoint 97-2003 (`.ppt`, `.pps`, `.pot`).", + "enum": [ + "ppt" + ], + "type": "string" + }, + { + "description": "PresentationML (`.pptx`, `.pptm`, `.ppsx`, `.ppsm`).", + "enum": [ + "pptx" + ], + "type": "string" + }, + { + "description": "OpenDocument Presentation (`.odp`).", + "enum": [ + "odp" + ], + "type": "string" + }, + { + "description": "Excel workbooks in every container (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`).", + "enum": [ + "excel" + ], + "type": "string" + }, + { + "description": "OpenDocument Spreadsheet (`.ods`).", + "enum": [ + "ods" + ], + "type": "string" + }, + { + "description": "Delimiter-separated text (`.csv`).", + "enum": [ + "csv" + ], + "type": "string" + }, + { + "description": "EPUB 2 and 3 (`.epub`).", + "enum": [ + "epub" + ], + "type": "string" + }, + { + "description": "Portable Document Format (`.pdf`).", + "enum": [ + "pdf" + ], + "type": "string" + } + ] + }, + "FsScope": { + "description": "The filesystem jail a call runs under.\n\nThe harness stamps this onto every function it dispatches, so a `path` an agent supplies has to be checked against it. Without the check these functions would read any document on the machine and hand back its text, which is a way around the scope the session was granted. Mirrors the shape the shell and pdf workers take.", + "properties": { + "grants": { + "default": [], + "description": "Additional directories or files explicitly granted to this session.", + "items": { + "type": "string" + }, + "type": "array" + }, + "root": { + "description": "The session's working directory.", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + } + }, + "description": "Where the document comes from. Exactly one of `path` and `bytes_base64` must be set.", + "properties": { + "bytes_base64": { + "default": null, + "description": "Base64-encoded document bytes, for a document with no path — an attachment held in memory. Mutually exclusive with `path`.", + "type": [ + "string", + "null" + ] + }, + "file_name": { + "default": null, + "description": "Original file name for inline bytes, used only to recognise a format the content cannot name. A `.csv` needs this; nothing else does. Ignored when `path` is set, which carries its own name.", + "type": [ + "string", + "null" + ] + }, + "format": { + "anyOf": [ + { + "$ref": "#/definitions/Format" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Force a format instead of detecting one. Only needed when the content carries no signature and the file name is absent or wrong." + }, + "fs_scope": { + "anyOf": [ + { + "$ref": "#/definitions/FsScope" + }, + { + "type": "null" + } + ], + "description": "The filesystem jail this call runs under. Stamped by the harness on an agent's call; absent on an operator or console call, which is already user-initiated and not subject to the agent's scope." + }, + "max_chars": { + "default": null, + "description": "Characters to return before truncating. Omit for the configured default; `0` returns the whole document.", + "format": "uint", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "path": { + "default": null, + "description": "Filesystem path to the document. Mutually exclusive with `bytes_base64`.", + "type": [ + "string", + "null" + ] + } + }, + "title": "Request", + "type": "object" + }, + "response_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "Body": { + "description": "A body that may have been shortened to fit one response, and the numbers a caller needs to decide what to do about it.\n\nThe cap is what keeps a long document from flooding a model's context. A caller that genuinely wants the whole thing asks for `max_chars: 0`, which is the shape a worker-to-worker pipeline uses to move a document without it passing through anyone's context. Same shape the pdf worker returns, so a caller handling both reads one field set.", + "properties": { + "chars": { + "description": "Characters returned in `text`.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "preview": { + "description": "Leading characters of the content. Present only when the body was truncated, so a caller can see the shape of what it did not get without re-reading the start of `text`.", + "type": [ + "string", + "null" + ] + }, + "text": { + "description": "The markdown, shortened to the effective character cap.", + "type": "string" + }, + "total_chars": { + "description": "Characters the document actually holds. Equal to `chars` when nothing was dropped.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "truncated": { + "description": "`true` when `text` stops short of the document. Ask again with `max_chars: 0` to take everything.", + "type": "boolean" + } + }, + "required": [ + "chars", + "text", + "total_chars", + "truncated" + ], + "type": "object" + }, + "DetectedFrom": { + "description": "How the format was arrived at, weakest claim last.", + "oneOf": [ + { + "description": "The caller named it, and the bytes were not consulted.", + "enum": [ + "requested" + ], + "type": "string" + }, + { + "description": "The signature the format's specification designates (PDF header, RTF open group, OLE stream names, ZIP package mimetype).", + "enum": [ + "content" + ], + "type": "string" + }, + { + "description": "The file extension only. CSV carries no signature, so this is the only way it is ever recognised; for any other format it means the content did not match anything known.", + "enum": [ + "extension" + ], + "type": "string" + } + ] + }, + "Family": { + "description": "What the document is, rather than which program wrote it.\n\nA caller routing a mixed bag of attachments cares that a file is a spreadsheet, not that it is `.ods` rather than `.xlsx`.", + "oneOf": [ + { + "description": "Prose: Word, OpenDocument Text, RTF.", + "enum": [ + "prose" + ], + "type": "string" + }, + { + "description": "Rows and columns: Excel, OpenDocument Spreadsheet, CSV.", + "enum": [ + "spreadsheet" + ], + "type": "string" + }, + { + "description": "Slides: PowerPoint, OpenDocument Presentation.", + "enum": [ + "presentation" + ], + "type": "string" + }, + { + "description": "A book: EPUB.", + "enum": [ + "book" + ], + "type": "string" + }, + { + "description": "PDF, which is its own family because it is the one format with a dedicated worker and a page-level OCR decision.", + "enum": [ + "pdf" + ], + "type": "string" + } + ] + }, + "Format": { + "description": "A format this worker converts. The names are the wire vocabulary: stable, lowercase, and independent of the file extension that named them (`.docm` is `docx`, `.xlsb` is `excel`).", + "oneOf": [ + { + "description": "Binary Word 97-2003 (`.doc`).", + "enum": [ + "doc" + ], + "type": "string" + }, + { + "description": "WordprocessingML (`.docx`, `.docm`).", + "enum": [ + "docx" + ], + "type": "string" + }, + { + "description": "OpenDocument Text (`.odt`).", + "enum": [ + "odt" + ], + "type": "string" + }, + { + "description": "Rich Text Format (`.rtf`).", + "enum": [ + "rtf" + ], + "type": "string" + }, + { + "description": "Binary PowerPoint 97-2003 (`.ppt`, `.pps`, `.pot`).", + "enum": [ + "ppt" + ], + "type": "string" + }, + { + "description": "PresentationML (`.pptx`, `.pptm`, `.ppsx`, `.ppsm`).", + "enum": [ + "pptx" + ], + "type": "string" + }, + { + "description": "OpenDocument Presentation (`.odp`).", + "enum": [ + "odp" + ], + "type": "string" + }, + { + "description": "Excel workbooks in every container (`.xlsx`, `.xlsm`, `.xlsb`, `.xls`).", + "enum": [ + "excel" + ], + "type": "string" + }, + { + "description": "OpenDocument Spreadsheet (`.ods`).", + "enum": [ + "ods" + ], + "type": "string" + }, + { + "description": "Delimiter-separated text (`.csv`).", + "enum": [ + "csv" + ], + "type": "string" + }, + { + "description": "EPUB 2 and 3 (`.epub`).", + "enum": [ + "epub" + ], + "type": "string" + }, + { + "description": "Portable Document Format (`.pdf`).", + "enum": [ + "pdf" + ], + "type": "string" + } + ] + } + }, + "properties": { + "asset_count": { + "description": "Embedded images and objects the document carries. Their bytes are not here — call `document::extract-assets` for those — but the count says whether a deck's content is pictures rather than text, which markdown alone would not reveal.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "body": { + "allOf": [ + { + "$ref": "#/definitions/Body" + } + ], + "description": "The markdown, capped per `max_chars`." + }, + "detected_from": { + "allOf": [ + { + "$ref": "#/definitions/DetectedFrom" + } + ], + "description": "How the format was arrived at." + }, + "elapsed_ms": { + "description": "Wall-clock time for the conversion.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "family": { + "allOf": [ + { + "$ref": "#/definitions/Family" + } + ], + "description": "What the document is: prose, a spreadsheet, a presentation, a book, a PDF." + }, + "format": { + "allOf": [ + { + "$ref": "#/definitions/Format" + } + ], + "description": "The format that was converted." + }, + "source": { + "description": "Source label: the file name, or `` for an in-memory document.", + "type": "string" + } + }, + "required": [ + "asset_count", + "body", + "detected_from", + "elapsed_ms", + "family", + "format", + "source" + ], + "title": "Response", + "type": "object" + } +} diff --git a/document/tests/schemas.rs b/document/tests/schemas.rs new file mode 100644 index 000000000..a0010d93f --- /dev/null +++ b/document/tests/schemas.rs @@ -0,0 +1,126 @@ +//! Wire-schema snapshots for the four `document::*` functions. +//! +//! `document::functions::catalog()` is the single source of truth for each +//! function's id, registration description, and schemars-derived request and +//! response schemas, generated with the same construction iii-sdk uses at +//! registration, from the same input and output structs. Each entry is +//! serialized to pretty JSON and compared against +//! `tests/golden/schemas/.json` (`::` maps to `.` in filenames). +//! +//! These snapshots ARE the product surface consumed by callers and agents, so +//! any schema or description change must land as an explicit golden diff. +//! Regenerate with `UPDATE_GOLDENS=1 cargo test`. + +mod support; + +use document::functions::{catalog, FunctionSpec}; + +fn golden_file_name(function_id: &str) -> String { + format!("schemas/{}.json", function_id.replace("::", ".")) +} + +fn spec_to_pretty_json(spec: &FunctionSpec) -> String { + let value = serde_json::json!({ + "function_id": spec.function_id, + "description": spec.description, + "request_schema": spec.request_schema, + "response_schema": spec.response_schema, + }); + let mut pretty = serde_json::to_string_pretty(&value).expect("spec serializes"); + pretty.push('\n'); + pretty +} + +/// The catalog must cover exactly the registered functions, in registration +/// order (kept in lockstep with `register_all`). +#[test] +fn catalog_lists_all_four_functions_in_registration_order() { + let ids: Vec<&str> = catalog().iter().map(|s| s.function_id).collect(); + assert_eq!( + ids, + vec![ + "document::detect", + "document::to-markdown", + "document::extract-assets", + "document::ocr", + ] + ); +} + +/// Every catalog entry matches its committed golden. Mismatches are collected +/// across ALL functions before failing, so one run shows the full drift. +#[test] +fn wire_schema_snapshots_match_goldens() { + let mut failures = Vec::new(); + for spec in catalog() { + let rel = golden_file_name(spec.function_id); + let actual = spec_to_pretty_json(&spec); + if let Err(msg) = support::check_golden(&rel, &actual) { + failures.push(msg); + } + } + assert!( + failures.is_empty(), + "{} wire-schema golden(s) drifted:\n\n{}", + failures.len(), + failures.join("\n") + ); +} + +/// No function may ship the permissive `AnyValue` schema — the deploy-time +/// "unknown" request/response schema this convention exists to prevent. +#[test] +fn every_function_has_typed_request_and_response_schemas() { + for spec in catalog() { + support::assert_typed_schema( + &format!("{} request_schema", spec.function_id), + &spec.request_schema, + ); + support::assert_typed_schema( + &format!("{} response_schema", spec.function_id), + &spec.response_schema, + ); + } +} + +/// Field doc comments become schema descriptions, and callers rely on them. +/// Losing them is a silent documentation regression that still compiles. +#[test] +fn schemas_carry_field_descriptions() { + for spec in catalog() { + let rendered = serde_json::to_string(&spec.request_schema).expect("schema serializes"); + assert!( + rendered.contains("description"), + "{}: request schema lost its field descriptions", + spec.function_id + ); + } +} + +/// Every function takes a document, and a caller holding bytes rather than a +/// path has to be able to see that from the schema alone. +#[test] +fn every_request_accepts_bytes_as_well_as_a_path() { + for spec in catalog() { + let rendered = serde_json::to_string(&spec.request_schema).expect("schema serializes"); + assert!( + rendered.contains("bytes_base64") && rendered.contains("\"path\""), + "{}: request must take either a path or inline bytes", + spec.function_id + ); + } +} + +/// The one convention a caller cannot guess: a CSV is only ever recognised by +/// its name, so the schema has to say the name matters. +#[test] +fn the_file_name_field_states_why_it_exists() { + for spec in catalog() { + let rendered = serde_json::to_string(&spec.request_schema).expect("schema serializes"); + assert!( + rendered.contains("file_name"), + "{}: inline bytes need a name for signature-less formats", + spec.function_id + ); + } +} diff --git a/document/tests/support/mod.rs b/document/tests/support/mod.rs new file mode 100644 index 000000000..d5e621d3d --- /dev/null +++ b/document/tests/support/mod.rs @@ -0,0 +1,119 @@ +//! Shared test support: the golden-file helpers behind `tests/schemas.rs`. +//! +//! Hand-rolled golden harness (deliberately no snapshot dependency). Goldens +//! live under `tests/golden/` and are committed; any wire-surface change must +//! show up as an explicit, reviewed diff. +//! +//! Workflow: +//! - `cargo test` compares actual output against the committed goldens. +//! - `UPDATE_GOLDENS=1 cargo test` regenerates the files; review the git diff, +//! then commit the new goldens alongside the change that caused them. + +#![allow(dead_code)] + +use std::fs; +use std::path::PathBuf; + +/// Root of the committed golden files. +pub fn golden_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/golden") +} + +fn update_mode() -> bool { + std::env::var("UPDATE_GOLDENS") + .map(|v| v == "1") + .unwrap_or(false) +} + +/// Compare `actual` against the golden file at `tests/golden/`. Returns +/// `Err(readable diff hint)` on mismatch or missing golden; with +/// `UPDATE_GOLDENS=1` the file is (re)written and the check passes. +pub fn check_golden(rel: &str, actual: &str) -> Result<(), String> { + let path = golden_root().join(rel); + if update_mode() { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; + } + fs::write(&path, actual).map_err(|e| format!("write {}: {e}", path.display()))?; + return Ok(()); + } + let expected = fs::read_to_string(&path).map_err(|e| { + format!( + "golden file {} unreadable ({e}).\n\ + Run `UPDATE_GOLDENS=1 cargo test` to (re)generate, then review and \ + commit the diff.", + path.display() + ) + })?; + if expected == actual { + return Ok(()); + } + Err(diff_hint(rel, &expected, actual)) +} + +/// Readable first-divergence diff hint: line number, expected versus actual +/// around the mismatch, and the regeneration instructions. +fn diff_hint(rel: &str, expected: &str, actual: &str) -> String { + let exp_lines: Vec<&str> = expected.lines().collect(); + let act_lines: Vec<&str> = actual.lines().collect(); + let first_diff = exp_lines + .iter() + .zip(act_lines.iter()) + .position(|(e, a)| e != a) + .unwrap_or_else(|| exp_lines.len().min(act_lines.len())); + + const CONTEXT: usize = 3; + let lo = first_diff.saturating_sub(CONTEXT); + let hi = (first_diff + CONTEXT + 1).max(first_diff + 1); + + let mut out = format!( + "golden mismatch: tests/golden/{rel}\n\ + first divergence at line {} (expected {} lines, actual {} lines)\n", + first_diff + 1, + exp_lines.len(), + act_lines.len() + ); + out.push_str("--- expected (golden) ---\n"); + for (i, line) in exp_lines.iter().enumerate().skip(lo).take(hi - lo) { + let marker = if i == first_diff { ">" } else { " " }; + out.push_str(&format!("{marker} {:>4} | {line}\n", i + 1)); + } + out.push_str("--- actual ---\n"); + for (i, line) in act_lines.iter().enumerate().skip(lo).take(hi - lo) { + let marker = if i == first_diff { ">" } else { " " }; + out.push_str(&format!("{marker} {:>4} | {line}\n", i + 1)); + } + out.push_str( + "If this change is intentional, run `UPDATE_GOLDENS=1 cargo test`, review \ + the git diff, and commit the updated goldens.\n", + ); + out +} + +/// Assert a schemars-derived request or response schema is a real schema and +/// not the permissive `AnyValue` schema a `Value` handler emits (the "unknown" +/// schema this convention exists to prevent). A real schema carries at least +/// one schema-defining keyword. +pub fn assert_typed_schema(label: &str, schema: &schemars::schema::RootSchema) { + let value = serde_json::to_value(schema).expect("schema serializes"); + let obj = value + .as_object() + .unwrap_or_else(|| panic!("{label}: schema is not a JSON object")); + const DEFINING: [&str; 8] = [ + "type", + "properties", + "$ref", + "allOf", + "anyOf", + "oneOf", + "enum", + "items", + ]; + let has_defining = DEFINING.iter().any(|k| obj.contains_key(*k)); + assert!( + has_defining, + "{label}: schema is the permissive AnyValue/empty schema (no \ + type/properties/$ref/…). The handler is registered with `Value` — give it \ + a typed struct deriving JsonSchema. Got: {value}" + ); +} diff --git a/iii-permissions.yaml b/iii-permissions.yaml index 7158cdb16..3d52111a7 100644 --- a/iii-permissions.yaml +++ b/iii-permissions.yaml @@ -195,6 +195,10 @@ rules: # other on-config-change denies. - '!pdf::on-config-change' + # document: internal target. The hot-reload hook follows the same pattern as + # the other on-config-change denies. + - '!document::on-config-change' + # canvas: internal target. The hot-reload hook follows the same pattern as # the other on-config-change denies. - '!canvas::on-config-change' @@ -349,6 +353,20 @@ rules: - pdf::extract-items - pdf::extract-regions + # document: same reasoning as pdf. Every function listed is a pure read of a + # file the agent could already reach through the filesystem scope, and reaching + # an office document any other way returns compressed noise. Nothing here + # writes, spends, or leaves the machine, and each response is capped by the + # worker's own configuration. + # + # `document::ocr` is deliberately absent: it renders pages and reads them with + # a vision model, so it spends money per page. It stays behind an approval, + # which is also what keeps an agent from transcribing a four-hundred-page scan + # because a document happened to be attached. + - document::detect + - document::to-markdown + - document::extract-assets + # canvas: every function touches only the worker's own store (state scope # "canvas"), sources are size-capped by its configuration, and nothing here # spends, executes, or leaves the machine. Storing and revising a diagram is diff --git a/pdf/README.md b/pdf/README.md index 6ab5ba989..a03ed2efe 100644 --- a/pdf/README.md +++ b/pdf/README.md @@ -151,6 +151,11 @@ It does not rasterize pages, so it cannot OCR anything. Scanned and image-based documents get classified and routed, not read. Image content is reported as a placeholder with a real bounding box and no pixels. +Routed where, in practice: [`document::ocr`](https://github.com/iii-hq/workers/tree/main/document) renders those pages +through the `browser` worker and reads them with a vision model. It costs money +per page, which is exactly why `pdf::classify` exists — pass it the +`pages_needing_ocr` named here rather than the whole document. + It is a parser, not a renderer: it walks the document's content streams and reconstructs the geometry, which is why it is fast and why it needs no service behind it. diff --git a/pdf/skills/SKILL.md b/pdf/skills/SKILL.md index 1e82ccd39..bff66caf2 100644 --- a/pdf/skills/SKILL.md +++ b/pdf/skills/SKILL.md @@ -49,7 +49,10 @@ pays for it. Reach for it when one appears. - Nothing here rasterizes a page, so nothing here can OCR. Scanned and image-based documents are classified and routed, never read. Image content - is reported as a placeholder with a real box and no pixels. + is reported as a placeholder with a real box and no pixels. `document::ocr` + is where routed pages go when the `document` worker is installed: it renders + them and reads them with a vision model, so it costs money per page. Hand it + the `pages_needing_ocr` this worker named rather than the whole file. - `suspected_garbled_text` in `ocr_reasons` means the text layer decodes to nonsense. Do not trust the extraction, whatever `document_type` says. - Responses are capped. `truncated: true` with a much larger `total_chars`