(MOT-4453) feat(document,console,browser): read every attachment kind, and take files by drag and paste

README.md

@@ -87,6 +87,7 @@ npx skills add iii-hq/iii --all
 | [`code-runner`](code-runner/) | Rust | Run untrusted Node.js and Python in-process — V8 isolates and CPython-on-WebAssembly behind one run/register_function/teardown API, with no microVM and no /dev/kvm. Code gets a global `iii` and a private scratch directory. |
 | [`openwiki`](openwiki/) | Node | Source-grounded markdown wiki for any git repository — a lead agent plans the index and writer sub-agents store cited pages via `openwiki::write-page`, with router and heuristic fallback tiers, incremental refresh from git diffs on a per-wiki cron schedule, and a browser UI + JSON API under `/openwiki`. |
 | [`pdf`](pdf/) | Rust | Read PDFs locally — `pdf::classify` routes text-based versus scanned in tens of milliseconds and names the pages that still need OCR, `pdf::to-markdown` converts with headings, lists and tables intact, and `pdf::extract-items` / `::extract-regions` expose positions and the text inside a box. Ships a console page. |
+| [`document`](document/) | Rust | Read office documents locally — `document::to-markdown` converts Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV and text-based PDFs with their structure intact, `document::detect` names a format from its bytes in microseconds, `document::extract-assets` returns the images markdown cannot carry, and `document::ocr` transcribes a scan by rendering its pages through `browser` and reading them with a vision model. |
 
 ## SDK
 

browser/README.md

@@ -158,11 +158,18 @@ browser:
   max_timeout_ms: 120000    # ceiling; caller timeout_ms clamped DOWN to this
   idle_stop_ms: 300000      # stop sessions idle this long; 0 disables
   screenshot_quality: 60    # JPEG quality 1-100
-  allowed_schemes: [http, https]
+  allowed_schemes: [http, https, file]  # `file` lets a local document be rendered; see below
   max_snapshot_nodes: 2000  # a11y outline size cap
   allow_attach: false       # true = allow sessions::attach into a running browser's real profile
 ```
 
+`file` is on the default scheme list so a local document can be opened and
+rendered, which is how `document::ocr` gets pixels out of a scanned PDF. It is
+worth knowing what that permits: navigation is not checked against a session's
+filesystem scope the way the workers that read files directly are, so anything
+that can reach `browser::navigate` can open any file this process can read.
+Narrow the list on a shared machine.
+
 ## Custom trigger types
 
 Sibling workers (and the console UI) can subscribe to session activity. All

browser/src/config.rs

@@ -47,6 +47,14 @@ pub struct WorkerConfig {
     /// JPEG quality for `browser::screenshot` (1-100).
     pub screenshot_quality: u64,
     /// URL schemes `browser::navigate` accepts.
+    ///
+    /// `file` ships enabled so a local document can be opened and rendered —
+    /// the path `document::ocr` takes to read a scanned PDF, which has nowhere
+    /// else to get pixels from. Note what that permits: unlike the workers that
+    /// read files directly, navigation is not checked against a session's
+    /// filesystem scope, so any caller that reaches `browser::navigate` can
+    /// open any file this process can read. Narrow the list on a shared or
+    /// multi-tenant machine.
     pub allowed_schemes: Vec<String>,
     /// Maximum nodes serialized by `browser::snapshot` before truncation.
     pub max_snapshot_nodes: u64,
@@ -71,7 +79,7 @@ impl Default for WorkerConfig {
             max_timeout_ms: 120_000,
             idle_stop_ms: 300_000,
             screenshot_quality: 60,
-            allowed_schemes: vec!["http".to_string(), "https".to_string()],
+            allowed_schemes: vec!["http".to_string(), "https".to_string(), "file".to_string()],
             max_snapshot_nodes: 2_000,
             allow_attach: false,
         }
@@ -128,7 +136,7 @@ mod tests {
         assert_eq!(c.max_timeout_ms, 120_000);
         assert_eq!(c.idle_stop_ms, 300_000);
         assert_eq!(c.screenshot_quality, 60);
-        assert_eq!(c.allowed_schemes, vec!["http", "https"]);
+        assert_eq!(c.allowed_schemes, vec!["http", "https", "file"]);
         assert_eq!(c.max_snapshot_nodes, 2_000);
         assert!(!c.allow_attach);
     }

browser/src/functions/sessions.rs

@@ -87,8 +87,22 @@ mod tests {
         let cfg = WorkerConfig::default();
         assert!(check_scheme(&cfg, "http://localhost:3000").is_ok());
         assert!(check_scheme(&cfg, "https://example.com/a?b=c").is_ok());
-        assert!(check_scheme(&cfg, "file:///etc/passwd").is_err());
+        // `file` ships enabled so a local document can be rendered.
+        assert!(check_scheme(&cfg, "file:///tmp/report.pdf").is_ok());
         assert!(check_scheme(&cfg, "chrome://settings").is_err());
         assert!(check_scheme(&cfg, "not a url").is_err());
     }
+
+    /// The list is what gates navigation, so an operator narrowing it has to
+    /// actually close the door — including on the scheme that now ships open.
+    #[test]
+    fn a_narrowed_list_still_refuses_what_it_drops() {
+        let cfg = WorkerConfig {
+            allowed_schemes: vec!["https".to_string()],
+            ..WorkerConfig::default()
+        };
+        assert!(check_scheme(&cfg, "https://example.com").is_ok());
+        assert!(check_scheme(&cfg, "file:///etc/passwd").is_err());
+        assert!(check_scheme(&cfg, "http://example.com").is_err());
+    }
 }

console/web/src/components/chat/AttachmentButton.tsx

@@ -1,6 +1,6 @@
 import { Paperclip } from 'lucide-react'
 import { useRef } from 'react'
-import { uid } from '@/hooks/use-conversations'
+import { attachmentsFromFiles } from '@/lib/attachments/from-files'
 import { cn } from '@/lib/utils'
 import type { Attachment } from '@/types/chat'
 
@@ -10,20 +10,6 @@ interface AttachmentButtonProps {
   className?: string
 }
 
-const MAX_PREVIEW_BYTES = 1_000_000
-
-function readPreview(file: File): Promise<string | undefined> {
-  if (file.size > MAX_PREVIEW_BYTES) return Promise.resolve(undefined)
-  if (!/^(image|text)\//.test(file.type)) return Promise.resolve(undefined)
-  return new Promise((resolve) => {
-    const reader = new FileReader()
-    reader.onload = () =>
-      resolve(typeof reader.result === 'string' ? reader.result : undefined)
-    reader.onerror = () => resolve(undefined)
-    reader.readAsDataURL(file)
-  })
-}
-
 export function AttachmentButton({
   onAttach,
   disabled,
@@ -34,19 +20,7 @@ export function AttachmentButton({
   const handlePick = async (e: React.ChangeEvent<HTMLInputElement>) => {
     const files = Array.from(e.target.files ?? [])
     if (files.length === 0) return
-    const attachments: Attachment[] = await Promise.all(
-      files.map(async (f) => ({
-        id: uid(),
-        name: f.name,
-        size: f.size,
-        type: f.type || 'application/octet-stream',
-        dataUrl: await readPreview(f),
-        // Kept so the send path can hand the bytes to a worker that reads this
-        // kind of file (PDFs go through `pdf::to-markdown`). Not persisted.
-        file: f,
-      })),
-    )
-    onAttach(attachments)
+    onAttach(await attachmentsFromFiles(files))
     /* allow re-picking the same file */
     e.target.value = ''
   }

console/web/src/components/chat/ChatView.tsx

@@ -23,9 +23,13 @@ import {
 import { useLiveAnnouncer } from '@/hooks/use-live-announcer'
 import { useWorktreeBinding } from '@/hooks/use-worktree-binding'
 import { useWorktreeEvents } from '@/hooks/use-worktree-events'
+import { expandAttachments, hasExpandableAttachments } from '@/lib/attachments'
 import type { ChatBackend } from '@/lib/backend'
 import { approvalBelongsToConversationTree } from '@/lib/backend/approval-events-live'
-import { predictedUserEntryId } from '@/lib/backend/harness-send'
+import {
+  type HarnessImageBlock,
+  predictedUserEntryId,
+} from '@/lib/backend/harness-send'
 import { serialRefresh } from '@/lib/backend/serial-refresh'
 import {
   mergeFiredTriggers,
@@ -41,11 +45,6 @@ import { useConversationsCtxOptional } from '@/lib/conversations-context'
 import { syncEditorWorkspace } from '@/lib/editor-sync'
 import { expandFileMentions, parseFileMentions } from '@/lib/file-mentions'
 import { formatStopReason } from '@/lib/format-stop-reason'
-import {
-  expandPdfAttachments,
-  isPdfAttachment,
-  summaryLabel,
-} from '@/lib/pdf-attachments'
 import { newMessageId } from '@/lib/session-id'
 import { useExtSessionChips, useExtSessionTurnSummaries } from '@/lib/ui-slots'
 import { cn } from '@/lib/utils'
@@ -210,6 +209,15 @@ export function ChatView({
   const harnessBlockedRef = useRef(harnessBlocked)
   harnessBlockedRef.current = harnessBlocked
 
+  /* What the model on the other end can do with a picture, read at send time
+     rather than closed over: the send and edit-queued callbacks are built
+     before the catalog lookup below, and a model switched between typing and
+     sending has to be the one the guard judges. Filled in further down. */
+  const visionRef = useRef<{ supports?: boolean; model: string | null }>({
+    supports: undefined,
+    model: null,
+  })
+
   // Live view of the transcript for the long-running stream loop: the
   // session-events reconciler (use-conversations) may add/replace rows while
   // a turn is in flight, and dedupe-by-functionTriggerId must see them.
@@ -580,17 +588,22 @@ export function ChatView({
             ).blocks
           }
         }
-        // Same expansion as the live send path: a queued message's PDFs have
-        // to reach the agent as markdown too, or editing a queued message
-        // would silently drop the document it carried.
+        // Same expansion as the live send path: a queued message's documents
+        // and pictures have to reach the agent too, or editing a queued
+        // message would silently drop what it carried.
+        let attachedImages: HarnessImageBlock[] | undefined
         if (
           backend.id === 'real' &&
-          payload.attachments.some(isPdfAttachment)
+          hasExpandableAttachments(payload.attachments)
         ) {
-          const expanded = await expandPdfAttachments(payload.attachments)
+          const expanded = await expandAttachments(payload.attachments, {
+            vision: visionRef.current.supports,
+            model: visionRef.current.model,
+          })
           if (expanded.blocks.length > 0) {
             attachedBlocks = [...(attachedBlocks ?? []), ...expanded.blocks]
           }
+          if (expanded.images.length > 0) attachedImages = expanded.images
           // Same reporting as the live send path. Staying silent here would let
           // an edited queued message lose its document with no explanation.
           for (const failure of expanded.failures) {
@@ -608,7 +621,9 @@ export function ChatView({
             conversationId,
             id,
             payload.text,
-            attachedBlocks ? { attachedBlocks } : undefined,
+            attachedBlocks || attachedImages
+              ? { attachedBlocks, attachedImages }
+              : undefined,
           )
         } catch (err) {
           onAppendMessage(
@@ -648,6 +663,16 @@ export function ChatView({
     return match?.contextWindow
   }, [modelOptions, effectiveModel])
 
+  /* What the send path may do with an attached picture. `undefined` when the
+     catalog has no row or the router said nothing — the attachment router
+     treats that as "send it", so a missing capability flag never silently
+     eats an image. */
+  const modelVision = useMemo(() => {
+    const match = modelOptions.find((o) => o.id === effectiveModel)
+    return match?.supportsVision
+  }, [modelOptions, effectiveModel])
+  visionRef.current = { supports: modelVision, model: effectiveModel }
+
   /* Injected session chips (the `chat` extension slot), rendered in the
    * header's right cluster where the built-in context meter sits. A chip
    * with id `context` supersedes the estimate-based ContextUsage meter —
@@ -1156,30 +1181,43 @@ export function ChatView({
         }
       }
 
-      // A PDF is not text: read as bytes it reaches the model as noise, so the
-      // `pdf` worker converts it on this machine and the markdown is appended
-      // as another attachment block. Failures never block the send — an
-      // unreadable document becomes a placeholder block plus a warn notice, so
-      // the model knows it was handed something it could not read.
-      if (backend.id === 'real' && payload.attachments.some(isPdfAttachment)) {
-        const expanded = await expandPdfAttachments(payload.attachments)
+      // Attachments are not text. A PDF or an office document read as bytes
+      // reaches the model as noise, and an image reaches it as nothing at all,
+      // so each kind is expanded on this machine first: documents into
+      // `<attached-file …>` markdown blocks, pictures into image content
+      // blocks. Failures never block the send — an unreadable attachment
+      // becomes a placeholder block plus a warn notice, so the model knows it
+      // was handed something that could not be read.
+      let attachedImages: HarnessImageBlock[] | undefined
+      if (
+        backend.id === 'real' &&
+        hasExpandableAttachments(payload.attachments)
+      ) {
+        const expanded = await expandAttachments(payload.attachments, {
+          vision: visionRef.current.supports,
+          model: visionRef.current.model,
+        })
         if (expanded.blocks.length > 0) {
           attachedBlocks = [...(attachedBlocks ?? []), ...expanded.blocks]
         }
-        // Relabel the chip with what the worker made of the document. The
-        // expansion runs before the model is called, so it never shows up as a
-        // function call — without this a person has no way to tell the PDF was
-        // read at all.
-        if (expanded.read.length > 0 && !willQueue) {
-          const byId = new Map(expanded.read.map((r) => [r.id, r]))
+        if (expanded.images.length > 0) attachedImages = expanded.images
+        // Drop the source bytes and relabel the chip with what the expansion
+        // made of each attachment. The relabel runs before the model is called,
+        // so it never shows up as a function call — without it a person has no
+        // way to tell the document was read at all.
+        //
+        // The `file` removal is NOT conditional on anything having been read:
+        // an attachment that failed, or an image refused for a model that
+        // cannot see, has finished its job too, and keeping its bytes would
+        // hold the whole file in memory for as long as the conversation stays
+        // open. Only the label depends on a matching entry.
+        if (!willQueue) {
+          const byId = new Map(expanded.read.map((r) => [r.id, r.label]))
           onPatchMessage(conversationId, userMsg.id, {
-            // `file` is dropped here as well as relabelled. It has done its job
-            // by now, and keeping it would hold the whole document in memory
-            // for as long as the conversation stays open.
             attachments: (userMsg.attachments ?? []).map(({ file, ...a }) => {
               void file
-              const summary = byId.get(a.id)
-              return summary ? { ...a, name: summaryLabel(a.name, summary) } : a
+              const label = byId.get(a.id)
+              return label ? { ...a, name: label } : a
             }),
           })
         }
@@ -1216,6 +1254,9 @@ export function ChatView({
               ...(attachedBlocks && attachedBlocks.length > 0
                 ? { attachedBlocks }
                 : {}),
+              ...(attachedImages && attachedImages.length > 0
+                ? { attachedImages }
+                : {}),
             },
           )
         } catch (err) {
@@ -1264,6 +1305,9 @@ export function ChatView({
             ...(attachedBlocks && attachedBlocks.length > 0
               ? { attachedBlocks }
               : {}),
+            ...(attachedImages && attachedImages.length > 0
+              ? { attachedImages }
+              : {}),
           },
         )) {
           switch (event.kind) {

console/web/src/components/chat/Composer.tsx

@@ -7,6 +7,7 @@ import {
 import { ArrowUp, Loader2, Square } from 'lucide-react'
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { PermissionModePicker } from '@/components/permissions/PermissionModePicker'
+import { attachmentsFromFiles } from '@/lib/attachments/from-files'
 import type { PermissionMode } from '@/lib/backend/approval-settings'
 import type { FunctionEntry } from '@/lib/functions'
 import { cn } from '@/lib/utils'
@@ -25,6 +26,7 @@ import { LexicalShell } from './LexicalShell'
 import { ModelPicker } from './ModelPicker'
 import { ModePicker } from './ModePicker'
 import { nextHistoryTarget } from './queue-history'
+import { useFileDrop } from './use-file-drop'
 
 export interface ComposerSubmitPayload {
   text: string
@@ -297,8 +299,38 @@ export function Composer({
     setAttachments((current) => current.filter((a) => a.id !== id))
   }, [])
 
+  const attachFiles = useCallback(
+    async (files: File[]) => {
+      if (files.length === 0) return
+      handleAttach(await attachmentsFromFiles(files))
+    },
+    [handleAttach],
+  )
+
+  // The drop zone is the whole chat pane, claimed in the capture phase — see
+  // `use-file-drop`. A drop onto the transcript, where people actually let go
+  // of a screenshot, lands here too, and the editor never gets to eat it.
+  const shell = useRef<HTMLDivElement>(null)
+  const dragging = useFileDrop({
+    anchorRef: shell,
+    disabled: Boolean(inputDisabled),
+    onFiles: (files) => void attachFiles(files),
+  })
+
   return (
-    <div className="rounded-xl bg-panel-raised shadow-raised">
+    <div
+      ref={shell}
+      className={cn(
+        'rounded-xl bg-panel-raised shadow-raised transition-shadow',
+        dragging && 'ring-2 ring-rule-focus',
+      )}
+    >
+      {dragging ? (
+        <div className="flex items-center justify-center border-b border-rule-2 px-3 py-2 font-mono text-[12px] text-ink-faint">
+          drop to attach
+        </div>
+      ) : null}
+
       {attachments.length > 0 ? (
         <div className="flex flex-wrap gap-2 p-3 border-b border-rule-2">
           {attachments.map((a) => (