diff --git a/companion/src/inbox.ts b/companion/src/inbox.ts
new file mode 100644
index 0000000000..914706d62b
--- /dev/null
+++ b/companion/src/inbox.ts
@@ -0,0 +1,158 @@
+// Files a paired phone drops onto this computer.
+//
+// The harness has no upload route. Desktop attachments are already-on-disk
+// paths (``); the agent opens them where it runs.
+// A photo on a phone is not on this disk, so the sidecar writes it here and
+// the phone sends that path as the message — the same shape every driver
+// already knows, and no harness change.
+//
+// Lives under the sidecar's own directory, not ~/.openmausbot. The two
+// processes do not share a layout; the agent still reads the file because
+// the path we return is an ordinary absolute path on this machine.
+import { randomBytes } from "node:crypto";
+import {
+ chmodSync,
+ closeSync,
+ constants,
+ fstatSync,
+ lstatSync,
+ mkdirSync,
+ openSync,
+ readSync,
+ writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { basename, join, resolve, sep } from "node:path";
+
+import { FILE_MODE } from "./state.ts";
+
+/** Eight megabytes. A phone photo after JPEG compression fits; a raw
+ * burst or a video does not, and this process should not hold one. */
+export const MAX_INBOX_BYTES = 8 * 1024 * 1024;
+
+export type StoredInboxFile = {
+ path: string;
+ name: string;
+ size: number;
+};
+
+/** Where new files land. Read at call time so tests can point `OMB_COMPANION_DIR`
+ * without reloading this module. */
+export function inboxRoot(): string {
+ const base = process.env.OMB_COMPANION_DIR ?? join(homedir(), ".openmausbot-companion");
+ return join(base, "inbox");
+}
+
+/** A filename that cannot walk out of the inbox. Basename only, a short
+ * allowlist of characters, no leading dots. Empty input becomes `file`. */
+export function safeFilename(input: string): string {
+ const base = basename(String(input ?? "")).replaceAll("\0", "");
+ const cleaned = base.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "");
+ return (cleaned || "file").slice(0, 80);
+}
+
+/** Write `bytes` into the inbox and return the path the phone should send. */
+export function storeInboxFile(
+ bytes: Buffer,
+ filename: string,
+ root = inboxRoot(),
+): StoredInboxFile {
+ if (bytes.length === 0) throw new Error("empty file");
+ if (bytes.length > MAX_INBOX_BYTES) throw new Error("body too large");
+
+ const name = safeFilename(filename);
+ mkdirSync(root, { recursive: true, mode: 0o700 });
+ try {
+ chmodSync(root, 0o700);
+ } catch {
+ /* existing dir on a filesystem that will not chmod — the write still works */
+ }
+
+ const stored = `${Date.now()}-${randomBytes(4).toString("hex")}-${name}`;
+ const path = join(root, stored);
+ const resolvedRoot = resolve(root);
+ const resolvedPath = resolve(path);
+ if (resolvedPath !== resolvedRoot && !resolvedPath.startsWith(resolvedRoot + sep)) {
+ throw new Error("invalid filename");
+ }
+
+ writeFileSync(path, bytes, { mode: FILE_MODE });
+ return { path, name, size: bytes.length };
+}
+
+/** Stored inbox names start with a digit (the timestamp). A leading dot is
+ * a traversal or a hidden file and is not one of ours. */
+const INBOX_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+
+/** Read a file the phone previously stored. Basename only, and only from
+ * this inbox — a stolen token must not become a reader for the rest of the
+ * disk. Missing or invalid names are `null`, not thrown. */
+export function readInboxFile(
+ filename: string,
+ root = inboxRoot(),
+): { bytes: Buffer; type: string } | null {
+ if (filename !== basename(filename) || !INBOX_NAME.test(filename)) return null;
+ const path = join(root, filename);
+ const resolvedRoot = resolve(root);
+ const resolvedPath = resolve(path);
+ if (resolvedPath !== resolvedRoot && !resolvedPath.startsWith(resolvedRoot + sep)) {
+ return null;
+ }
+ let fd: number | undefined;
+ try {
+ // Open once, then fstat/read that descriptor. `lstat` then `readFile`
+ // is two path lookups: a symlink swapped in between them would follow
+ // out of the inbox. `O_NOFOLLOW` is POSIX; Windows does not expose it
+ // (or exposes `0`), so a reparse point is refused with `lstat` first.
+ // That is still a path check — not the same as no-follow open — but
+ // following here would read outside the inbox.
+ const noFollow = constants.O_NOFOLLOW;
+ const hasNoFollow = typeof noFollow === "number" && noFollow !== 0;
+ if (!hasNoFollow && lstatSync(path).isSymbolicLink()) return null;
+ const flags = hasNoFollow ? constants.O_RDONLY | noFollow : constants.O_RDONLY;
+ fd = openSync(path, flags);
+ const st = fstatSync(fd);
+ if (!st.isFile() || st.size > MAX_INBOX_BYTES) return null;
+ // Read at most the size we just validated. `readFileSync(fd)` would
+ // follow a concurrent append past MAX_INBOX_BYTES.
+ const bytes = Buffer.alloc(st.size);
+ let offset = 0;
+ while (offset < bytes.length) {
+ const n = readSync(fd, bytes, offset, bytes.length - offset, offset);
+ if (n === 0) break;
+ offset += n;
+ }
+ return { bytes: offset === bytes.length ? bytes : bytes.subarray(0, offset), type: inboxType(filename) };
+ } catch {
+ return null;
+ } finally {
+ if (fd !== undefined) {
+ try {
+ closeSync(fd);
+ } catch {
+ /* already closed */
+ }
+ }
+ }
+}
+
+export function inboxType(filename: string): string {
+ switch (filename.split(".").pop()?.toLowerCase()) {
+ case "jpg":
+ case "jpeg":
+ return "image/jpeg";
+ case "png":
+ return "image/png";
+ case "gif":
+ return "image/gif";
+ case "webp":
+ return "image/webp";
+ case "heic":
+ case "heif":
+ return "image/heic";
+ case "pdf":
+ return "application/pdf";
+ default:
+ return "application/octet-stream";
+ }
+}
diff --git a/companion/src/proxy.ts b/companion/src/proxy.ts
index 22c7671a8c..2e4475a26e 100644
--- a/companion/src/proxy.ts
+++ b/companion/src/proxy.ts
@@ -14,6 +14,7 @@
import { request as httpRequest, type IncomingMessage, type ServerResponse } from "node:http";
import { bearerToken } from "./devices.ts";
+import { MAX_INBOX_BYTES, readInboxFile, storeInboxFile } from "./inbox.ts";
import { denyReason, isCloudDesktopJoin } from "./routes.ts";
import { createSseScrubber, isJson, scrub } from "./wire.ts";
@@ -83,6 +84,38 @@ const readJson = (req: IncomingMessage, limit = 64 * 1024): Promise =>
+ new Promise((resolve, reject) => {
+ let size = 0;
+ let settled = false;
+ const chunks: Buffer[] = [];
+ const fail = (error: Error) => {
+ if (settled) return;
+ settled = true;
+ reject(error);
+ };
+ req.on("data", (chunk: Buffer) => {
+ if (settled) return;
+ size += chunk.length;
+ if (size > limit) {
+ fail(new Error("body too large"));
+ return;
+ }
+ chunks.push(chunk);
+ });
+ req.on("error", fail);
+ req.on("end", () => {
+ if (settled) return;
+ settled = true;
+ resolve(Buffer.concat(chunks));
+ });
+ });
+
/** Answer with JSON the sidecar wrote itself — a refusal, or a pairing
* result. Anything from the harness goes out through the proxy path instead.
*
@@ -185,6 +218,59 @@ export function createProxyHandler(options: ProxyOptions) {
return;
}
+ // Phone attachments terminate here. Forwarding them would hand the
+ // harness a route it does not have. The sidecar writes the bytes onto
+ // this computer and returns the path; the next request is an ordinary
+ // text message carrying ``.
+ if (method === "POST" && path === "/api/inbox") {
+ readBytes(req, MAX_INBOX_BYTES).then(
+ (bytes) => {
+ try {
+ const header = req.headers["x-openmaus-filename"];
+ const raw = Array.isArray(header) ? header[0] : header;
+ let filename = "file";
+ try {
+ filename = decodeURIComponent(String(raw ?? "file"));
+ } catch {
+ filename = String(raw ?? "file");
+ }
+ return sendJson(res, 201, storeInboxFile(bytes, filename));
+ } catch (error) {
+ const message = error instanceof Error ? error.message : "could not store that file";
+ return sendJson(res, message.includes("too large") ? 413 : 400, { error: message });
+ }
+ },
+ (error: Error) => {
+ sendJson(res, error.message === "body too large" ? 413 : 400, { error: error.message });
+ // Drain whatever is still in flight so the 413 is not sitting
+ // behind a half-read body. Destroying the socket here is how a
+ // client sees a dropped connection instead of that status.
+ req.resume();
+ },
+ );
+ return;
+ }
+
+ if (method === "GET") {
+ const match = /^\/api\/inbox\/([^/]+)$/.exec(path);
+ if (match) {
+ let name = match[1];
+ try {
+ name = decodeURIComponent(name);
+ } catch {
+ return sendJson(res, 400, { error: "invalid filename" });
+ }
+ const stored = readInboxFile(name);
+ if (!stored) return sendJson(res, 404, { error: "no such file" });
+ res.writeHead(200, {
+ "content-type": stored.type,
+ "content-length": stored.bytes.length,
+ });
+ res.end(stored.bytes);
+ return;
+ }
+ }
+
const upstream = httpRequest(
{
hostname: "127.0.0.1",
diff --git a/companion/src/routes.ts b/companion/src/routes.ts
index 7ca9dbb9b2..6a9d2c3f8f 100644
--- a/companion/src/routes.ts
+++ b/companion/src/routes.ts
@@ -85,6 +85,12 @@ const ALLOWED: ReadonlyArray<{ method: string; path: RegExp }> = [
{ method: "GET", path: /^\/api\/threads\/[\w-]+\/export$/ },
{ method: "POST", path: /^\/api\/threads\/[\w-]+\/respond$/ },
{ method: "GET", path: /^\/api\/search$/ },
+
+ // Phone photos and files. Handled by the sidecar itself — the harness
+ // has no upload route. The phone then sends the returned host path as
+ // a normal text message, the same shape the desktop composer already uses.
+ { method: "POST", path: /^\/api\/inbox$/ },
+ { method: "GET", path: /^\/api\/inbox\/[A-Za-z0-9][\w.-]*$/ },
];
/** Route families worth naming in the refusal.
diff --git a/companion/test/inbox.test.ts b/companion/test/inbox.test.ts
new file mode 100644
index 0000000000..65a1fcb7d8
--- /dev/null
+++ b/companion/test/inbox.test.ts
@@ -0,0 +1,94 @@
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+
+import { MAX_INBOX_BYTES, readInboxFile, safeFilename, storeInboxFile } from "../src/inbox.ts";
+
+let root: string | undefined;
+
+afterEach(() => {
+ if (root) rmSync(root, { recursive: true, force: true });
+ root = undefined;
+});
+
+const dir = () => (root = mkdtempSync(join(tmpdir(), "inbox-")));
+
+describe("safeFilename", () => {
+ it("keeps a plain name", () => {
+ expect(safeFilename("notes.txt")).toBe("notes.txt");
+ });
+
+ it("strips directory components so a traversal cannot leave the inbox", () => {
+ expect(safeFilename("../../etc/passwd")).toBe("passwd");
+ expect(safeFilename("foo/bar/photo.jpg")).toBe("photo.jpg");
+ });
+
+ it("replaces characters a path would treat specially", () => {
+ expect(safeFilename("my photo (1).jpg")).toBe("my_photo_1_.jpg");
+ expect(safeFilename("a\0b.png")).toBe("ab.png");
+ });
+
+ it("does not keep a leading dot, and empty input is still a name", () => {
+ expect(safeFilename("")).toBe("file");
+ expect(safeFilename("...")).toBe("file");
+ expect(safeFilename(".hidden")).toBe("hidden");
+ });
+
+ it("caps a long name so the inbox path stays short", () => {
+ expect(safeFilename("a".repeat(200)).length).toBe(80);
+ });
+});
+
+describe("storeInboxFile", () => {
+ it("writes the bytes and returns an absolute path inside the root", () => {
+ const stored = storeInboxFile(Buffer.from("hello"), "notes.txt", dir());
+ expect(stored.name).toBe("notes.txt");
+ expect(stored.size).toBe(5);
+ expect(stored.path.startsWith(root!)).toBe(true);
+ expect(readFileSync(stored.path, "utf8")).toBe("hello");
+ });
+
+ it("refuses an empty body and a body over the ceiling", () => {
+ expect(() => storeInboxFile(Buffer.alloc(0), "a.txt", dir())).toThrow(/empty/);
+ expect(() => storeInboxFile(Buffer.alloc(MAX_INBOX_BYTES + 1), "a.bin", dir())).toThrow(
+ /too large/,
+ );
+ });
+
+ it("does not collide when the same name is stored twice", () => {
+ const a = storeInboxFile(Buffer.from("one"), "same.txt", dir());
+ const b = storeInboxFile(Buffer.from("two"), "same.txt", root!);
+ expect(a.path).not.toBe(b.path);
+ expect(readFileSync(a.path, "utf8")).toBe("one");
+ expect(readFileSync(b.path, "utf8")).toBe("two");
+ });
+
+ it("reads a stored file back, and refuses a name that could leave the inbox", () => {
+ const stored = storeInboxFile(Buffer.from("hello"), "notes.txt", dir());
+ const got = readInboxFile(basename(stored.path), root!);
+ expect(got?.bytes.toString("utf8")).toBe("hello");
+ expect(got?.type).toBe("application/octet-stream");
+ expect(readInboxFile("../notes.txt", root!)).toBeNull();
+ expect(readInboxFile("..", root!)).toBeNull();
+ expect(readInboxFile(".hidden", root!)).toBeNull();
+ expect(readInboxFile("missing.txt", root!)).toBeNull();
+ });
+
+ it("refuses a symlink, a directory, and a body over the ceiling", () => {
+ const root = dir();
+ writeFileSync(join(root, "target.txt"), "secret");
+ const name = "1-abcd1234-notes.txt";
+ try {
+ symlinkSync("target.txt", join(root, name));
+ expect(readInboxFile(name, root)).toBeNull();
+ } catch {
+ // some CI filesystems will not make a symlink; the other two
+ // refusals below still cover the same read path
+ }
+ mkdirSync(join(root, "2-abcd1234-dir"));
+ expect(readInboxFile("2-abcd1234-dir", root)).toBeNull();
+ writeFileSync(join(root, "3-abcd1234-big.bin"), Buffer.alloc(MAX_INBOX_BYTES + 1));
+ expect(readInboxFile("3-abcd1234-big.bin", root)).toBeNull();
+ });
+});
diff --git a/companion/test/proxy.test.ts b/companion/test/proxy.test.ts
index 56de9be278..2e9d0c8d9d 100644
--- a/companion/test/proxy.test.ts
+++ b/companion/test/proxy.test.ts
@@ -7,9 +7,9 @@
// through, and the harness's loopback gate rejecting a proxied request.
import { spawn, type ChildProcess } from "node:child_process";
import { createServer, request, type Server } from "node:http";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
+import { dirname, basename, join } from "node:path";
import { fileURLToPath } from "node:url";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
@@ -71,6 +71,7 @@ const TOKEN = "omb_test_token";
let harness: ChildProcess;
let sidecar: Server;
let home: string;
+let previousCompanionDir: string | undefined;
let stderr = "";
/** a request as a device makes it: a token, and a Host that is not loopback */
@@ -127,6 +128,8 @@ beforeAll(async () => {
SIDECAR = `http://127.0.0.1:${SIDECAR_PORT}`;
home = mkdtempSync(join(tmpdir(), "companion-test-"));
+ previousCompanionDir = process.env.OMB_COMPANION_DIR;
+ process.env.OMB_COMPANION_DIR = join(home, "companion-data");
mkdirSync(join(home, ".openmausbot"), { recursive: true });
writeFileSync(
join(home, ".openmausbot", "config.json"),
@@ -201,6 +204,8 @@ afterAll(async () => {
setTimeout(resolve, 10_000).unref?.();
});
rmSync(home, { recursive: true, force: true });
+ if (previousCompanionDir === undefined) delete process.env.OMB_COMPANION_DIR;
+ else process.env.OMB_COMPANION_DIR = previousCompanionDir;
});
describe("the sidecar in front of an unmodified harness", () => {
@@ -235,6 +240,50 @@ describe("the sidecar in front of an unmodified harness", () => {
expect((await device("GET", "/api/bots")).status).toBe(200);
});
+ it("writes an uploaded file onto this computer and returns a path the bot can open", async () => {
+ const payload = Buffer.from("hello from the phone");
+ const res = await fetch(`${SIDECAR}/api/inbox`, {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${TOKEN}`,
+ "content-type": "application/octet-stream",
+ "x-openmaus-filename": encodeURIComponent("notes.txt"),
+ },
+ body: payload,
+ });
+ expect(res.status).toBe(201);
+ const body = (await res.json()) as { path: string; name: string; size: number };
+ expect(body.name).toBe("notes.txt");
+ expect(body.size).toBe(payload.length);
+ expect(body.path.startsWith(join(home, "companion-data", "inbox"))).toBe(true);
+ expect(readFileSync(body.path, "utf8")).toBe("hello from the phone");
+
+ const stored = basename(body.path);
+ const got = await fetch(`${SIDECAR}/api/inbox/${stored}`, {
+ headers: { authorization: `Bearer ${TOKEN}` },
+ });
+ expect(got.status).toBe(200);
+ expect(got.headers.get("content-type")).toBe("application/octet-stream");
+ expect(Buffer.from(await got.arrayBuffer()).toString("utf8")).toBe("hello from the phone");
+ expect((await device("GET", `/api/inbox/${stored}`, { token: null })).status).toBe(401);
+ expect((await device("GET", "/api/inbox/no-such-file.txt")).status).toBe(404);
+ });
+
+ it("refuses an inbox upload without a token, and a body over the ceiling", async () => {
+ expect((await device("POST", "/api/inbox", { token: null })).status).toBe(401);
+ expect((await device("GET", "/api/inbox")).status).toBe(404);
+ const res = await fetch(`${SIDECAR}/api/inbox`, {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${TOKEN}`,
+ "content-type": "application/octet-stream",
+ "x-openmaus-filename": "too-big.bin",
+ },
+ body: Buffer.alloc(8 * 1024 * 1024 + 1),
+ });
+ expect(res.status).toBe(413);
+ });
+
it("refuses what a device has no business doing, by default", async () => {
// settings and credentials stay on the machine
expect((await device("PUT", "/api/config", { body: { xai: { apiKey: "x" } } })).status).toBe(403);
diff --git a/companion/test/routes.test.ts b/companion/test/routes.test.ts
index 08d3905563..d8877d0dbb 100644
--- a/companion/test/routes.test.ts
+++ b/companion/test/routes.test.ts
@@ -57,6 +57,8 @@ describe("what the app may do", () => {
["GET", "/api/threads/th_1/export"],
["POST", "/api/threads/th_1/respond"],
["GET", "/api/search"],
+ ["POST", "/api/inbox"],
+ ["GET", "/api/inbox/178-ab-photo.jpg"],
];
for (const [method, path] of calls) {
@@ -109,6 +111,7 @@ describe("what it may not", () => {
it("allows a path only for the methods it was allowed for", () => {
expect(allowed("GET", "/api/bots")).toBe(true);
expect(allowed("DELETE", "/api/bots/bot_123")).toBe(false);
+ expect(allowed("GET", "/api/inbox")).toBe(false);
expect(allowed("POST", "/api/threads/th_1/messages")).toBe(false);
expect(allowed("GET", "/api/groups/room-1")).toBe(false);
expect(allowed("PATCH", "/api/bots/bot_123")).toBe(false);
@@ -122,6 +125,8 @@ describe("what it may not", () => {
expect(allowed("GET", "/api/botsandthensome")).toBe(false);
expect(allowed("GET", "/api/events/all")).toBe(false);
expect(allowed("GET", "/api/threads/th_1/messages/msg_2/image/../../../config")).toBe(false);
+ expect(allowed("GET", "/api/inbox/../passwd")).toBe(false);
+ expect(allowed("GET", "/api/inbox/..")).toBe(false);
expect(allowed("GET", "/api/bots%2f..%2fwebhooks")).toBe(false);
});
diff --git a/docs/ios-companion.md b/docs/ios-companion.md
index f3f6f4d2fa..c61eedae66 100644
--- a/docs/ios-companion.md
+++ b/docs/ios-companion.md
@@ -20,9 +20,11 @@ The first version includes:
- Resumable SSE, streamed reply text, reconnect hydration, and an opt-in live
computer view.
- Markdown rendering and Keychain storage for the device token.
+- Composer dictation, and attaching a photo or file (written onto the
+ computer, then sent as the same path tag the desktop composer uses).
-It is foreground-only. Push notifications, background delivery, voice, App
-Store release automation, and a hosted relay are not part of this version.
+It is foreground-only. Push notifications, background delivery, call mode,
+App Store release automation, and a hosted relay are not part of this version.
## Runtime architecture
@@ -190,6 +192,7 @@ companion/
src/routes.ts device-facing allowlist
src/devices.ts pairing and token registry
src/proxy.ts HTTP/SSE forwarding and scrubbing
+ src/inbox.ts phone files written onto this computer
src/control.ts loopback-only control plane
src/mdns.ts Bonjour advertisement
diff --git a/ios/App/CameraPicker.swift b/ios/App/CameraPicker.swift
new file mode 100644
index 0000000000..c1b432ef95
--- /dev/null
+++ b/ios/App/CameraPicker.swift
@@ -0,0 +1,48 @@
+// Camera capture for the composer. PhotosPicker covers the library;
+// taking a new picture is still UIImagePickerController.
+import SwiftUI
+import UIKit
+
+struct CameraPicker: UIViewControllerRepresentable {
+ var onImage: (UIImage) -> Void
+ var onCancel: () -> Void
+
+ func makeCoordinator() -> Coordinator {
+ Coordinator(onImage: onImage, onCancel: onCancel)
+ }
+
+ func makeUIViewController(context: Context) -> UIImagePickerController {
+ let picker = UIImagePickerController()
+ picker.sourceType = .camera
+ picker.cameraCaptureMode = .photo
+ picker.delegate = context.coordinator
+ return picker
+ }
+
+ func updateUIViewController(_ uiViewController: UIImagePickerController, context: Context) {}
+
+ final class Coordinator: NSObject, UIImagePickerControllerDelegate, UINavigationControllerDelegate {
+ let onImage: (UIImage) -> Void
+ let onCancel: () -> Void
+
+ init(onImage: @escaping (UIImage) -> Void, onCancel: @escaping () -> Void) {
+ self.onImage = onImage
+ self.onCancel = onCancel
+ }
+
+ func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
+ onCancel()
+ }
+
+ func imagePickerController(
+ _ picker: UIImagePickerController,
+ didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey: Any]
+ ) {
+ if let image = info[.originalImage] as? UIImage {
+ onImage(image)
+ } else {
+ onCancel()
+ }
+ }
+ }
+}
diff --git a/ios/App/ChatView.swift b/ios/App/ChatView.swift
index 33e8a5b651..bcc50e980d 100644
--- a/ios/App/ChatView.swift
+++ b/ios/App/ChatView.swift
@@ -14,15 +14,27 @@ import CompanionCore
// isn't. The App target is iOS; CompanionCore is where the portable half
// lives.
import UIKit
+import AVFoundation
+import PhotosUI
+import UniformTypeIdentifiers
struct ChatView: View {
let chat: Chat
@EnvironmentObject private var session: Session
@Environment(\.dismiss) private var dismiss
+ @Environment(\.scenePhase) private var scenePhase
@State private var draft = ""
@State private var showingTasks = false
@State private var shareFile: ShareFile?
@FocusState private var composerFocused: Bool
+ @StateObject private var dictation = SpeechDictation()
+ @State private var pending: [PendingAttachment] = []
+ @State private var attachError: String?
+ @State private var pickingPhotos = false
+ @State private var photoItems: [PhotosPickerItem] = []
+ @State private var pickingCamera = false
+ @State private var pickingFiles = false
+ @State private var sendingAttachments = false
/// The live bubble's scroll target. A constant because there is at most
/// one per chat and it has no message id to borrow.
@@ -181,7 +193,11 @@ struct ChatView: View {
// guess. Bots only.
ToolbarItem(placement: .topBarTrailing) {
NavigationLink {
+ // Pushing does not disappear ChatView — it stays in
+ // the stack under the computer panel — so onDisappear
+ // would leave the mic open behind another screen.
ComputerView(bot: bot)
+ .onAppear { dictation.stop() }
} label: {
Image(systemName: "display")
.font(.system(size: 15, weight: .medium))
@@ -231,6 +247,29 @@ struct ChatView: View {
// bit here rather than leaving a badge on an open conversation.
if unread { Task { await session.markRead(current) } }
}
+ .onDisappear { dictation.stop() }
+ // Backgrounding does not always disappear this view — it stays in
+ // the navigation stack — and a microphone left open through a lock
+ // is a privacy surprise. The stream is already torn down on
+ // inactive; dictation should follow.
+ .onChange(of: scenePhase) { _, phase in
+ if phase != .active { dictation.stop() }
+ }
+ .onReceive(NotificationCenter.default.publisher(for: AVAudioSession.interruptionNotification)) { note in
+ let raw = note.userInfo?[AVAudioSessionInterruptionTypeKey]
+ let value = (raw as? NSNumber)?.uintValue ?? (raw as? UInt)
+ if value == AVAudioSession.InterruptionType.began.rawValue {
+ dictation.stop()
+ }
+ }
+ // Frozen `dictation.base`, not the live draft: a later partial
+ // replaces an earlier one rather than concatenating onto it.
+ .onChange(of: dictation.transcript) { _, spoken in
+ draft = Dictation.draft(base: dictation.base, transcript: spoken)
+ }
+ .onChange(of: dictation.isListening) { _, listening in
+ if listening { composerFocused = false }
+ }
.sheet(isPresented: $showingTasks) {
if case let .bot(bot) = current { TaskManagerView(bot: bot) }
}
@@ -247,25 +286,215 @@ struct ChatView: View {
}
private var canSend: Bool {
- !draft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ sendingAttachments == false
+ && (
+ !draft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ || !pending.isEmpty
+ )
}
private func submit() {
+ // Always, not only when `isListening`: send during the permission
+ // prompt must cancel the in-flight start, or capture would begin
+ // after the message has already left.
+ dictation.stop()
let text = draft.trimmingCharacters(in: .whitespacesAndNewlines)
- guard !text.isEmpty else { return }
- draft = ""
- Task { await session.send(text, to: current) }
+ let ids = pending.map(\.id)
+ guard !text.isEmpty || !ids.isEmpty, !sendingAttachments else { return }
+ sendingAttachments = true
+ attachError = nil
+ Task {
+ var files: [Attachment.File] = []
+ for id in ids {
+ guard let index = pending.firstIndex(where: { $0.id == id }) else { continue }
+ if let host = pending[index].host {
+ if let preview = pending[index].preview {
+ session.rememberPreview(preview, for: host.path)
+ }
+ InboxCache.save(pending[index].data, hostPath: host.path)
+ files.append(host)
+ continue
+ }
+ let item = pending[index]
+ guard let stored = await session.upload(item.data, filename: item.name) else {
+ sendingAttachments = false
+ attachError = session.actionError ?? "Couldn't send that file."
+ return
+ }
+ guard let latest = pending.firstIndex(where: { $0.id == id }) else { continue }
+ let file = Attachment.File(path: stored.path, name: stored.name, size: stored.size)
+ pending[latest].host = file
+ if let preview = pending[latest].preview {
+ session.rememberPreview(preview, for: file.path)
+ }
+ InboxCache.save(item.data, hostPath: file.path)
+ files.append(file)
+ }
+ let body = Attachment.draft(text: text, files: files)
+ guard !body.isEmpty else {
+ sendingAttachments = false
+ return
+ }
+ let sent = await session.send(body, to: current)
+ sendingAttachments = false
+ guard sent else {
+ attachError = session.actionError ?? "Couldn't send that."
+ return
+ }
+ if draft.trimmingCharacters(in: .whitespacesAndNewlines) == text {
+ draft = ""
+ }
+ pending.removeAll { ids.contains($0.id) }
+ }
+ }
+
+ private func addPhoto(_ image: UIImage, name: String = "photo.jpg") {
+ guard pending.count < PendingMedia.maxCount else {
+ attachError = "You can attach up to \(PendingMedia.maxCount) files."
+ return
+ }
+ guard let item = PendingMedia.jpegAttachment(from: image, name: uniqueName(name)) else {
+ attachError = "That image is too large to send."
+ return
+ }
+ attachError = nil
+ pending.append(item)
+ }
+
+ private func addFile(name: String, data: Data, preview: UIImage? = nil) {
+ guard !data.isEmpty else { return }
+ guard pending.count < PendingMedia.maxCount else {
+ attachError = "You can attach up to \(PendingMedia.maxCount) files."
+ return
+ }
+ guard data.count <= PendingMedia.maxBytes else {
+ attachError = "\(name) is larger than 8 MB."
+ return
+ }
+ attachError = nil
+ pending.append(PendingAttachment(name: uniqueName(name), data: data, preview: preview))
+ }
+
+ /// Chip labels stay distinct when two photos would otherwise both be `photo.jpg`.
+ private func uniqueName(_ name: String) -> String {
+ if !pending.contains(where: { $0.name == name }) { return name }
+ let ns = name as NSString
+ let ext = ns.pathExtension
+ let stem = ext.isEmpty ? name : ns.deletingPathExtension
+ for n in 2...(PendingMedia.maxCount + 1) {
+ let candidate = ext.isEmpty ? "\(stem)-\(n)" : "\(stem)-\(n).\(ext)"
+ if !pending.contains(where: { $0.name == candidate }) { return candidate }
+ }
+ return name
+ }
+
+ private func consumePhotos(_ items: [PhotosPickerItem]) async {
+ for item in items {
+ if let data = try? await item.loadTransferable(type: Data.self),
+ let image = UIImage(data: data) {
+ addPhoto(image, name: "photo.jpg")
+ } else {
+ attachError = "Couldn't read that photo."
+ }
+ }
+ photoItems = []
+ }
+
+ private func consumeFiles(_ urls: [URL]) {
+ for url in urls {
+ let accessed = url.startAccessingSecurityScopedResource()
+ defer { if accessed { url.stopAccessingSecurityScopedResource() } }
+ var isDirectory: ObjCBool = false
+ let exists = FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory)
+ guard exists, !isDirectory.boolValue, !url.hasDirectoryPath else {
+ attachError = "\(url.lastPathComponent) is a folder."
+ continue
+ }
+ let listedSize = (try? url.resourceValues(forKeys: [.fileSizeKey]))?.fileSize
+ if let listedSize, listedSize > PendingMedia.maxBytes {
+ attachError = "\(url.lastPathComponent) is larger than 8 MB."
+ continue
+ }
+ guard let data = readAtMost(PendingMedia.maxBytes, from: url) else {
+ attachError = "Couldn't open \(url.lastPathComponent)."
+ continue
+ }
+ if data.count > PendingMedia.maxBytes {
+ attachError = "\(url.lastPathComponent) is larger than 8 MB."
+ continue
+ }
+ addFile(name: url.lastPathComponent, data: data, preview: PendingMedia.thumbnail(from: data))
+ }
+ }
+
+ /// Read at most `limit + 1` bytes so an oversized file is rejected
+ /// without being fully loaded. `fileSizeKey` is checked first; this
+ /// covers a missing size or a file that grew after that listing.
+ private func readAtMost(_ limit: Int, from url: URL) -> Data? {
+ guard let handle = try? FileHandle(forReadingFrom: url) else { return nil }
+ defer { try? handle.close() }
+ return try? handle.read(upToCount: limit + 1)
}
private var composer: some View {
- HStack(spacing: 10) {
- TextField("Ask \(current.name)", text: $draft, axis: .vertical)
+ VStack(spacing: 6) {
+ if let error = dictation.error {
+ Text(error)
+ .font(.system(size: 13))
+ .foregroundStyle(.orange)
+ .frame(maxWidth: .infinity, alignment: .leading)
+ .padding(.horizontal, 4)
+ }
+ if attachError != nil || !pending.isEmpty {
+ ComposerAttachBar(items: $pending, error: attachError, enabled: !sendingAttachments)
+ }
+ // Bottom, not centre: a wrapping field used to grow into a
+ // stadium with the mic and send floating at its middle. Other
+ // chat apps pin the actions to the last line.
+ HStack(alignment: .bottom, spacing: 10) {
+ ComposerAttachMenu(
+ enabled: !dictation.isListening && !sendingAttachments,
+ onAttachImage: {
+ dictation.stop()
+ pickingPhotos = true
+ },
+ onTakePhoto: {
+ dictation.stop()
+ guard UIImagePickerController.isSourceTypeAvailable(.camera) else {
+ attachError = "This device has no camera."
+ return
+ }
+ pickingCamera = true
+ },
+ onChooseFile: {
+ dictation.stop()
+ pickingFiles = true
+ }
+ )
+ TextField(
+ dictation.isListening ? "Listening…" : "Ask \(current.name)",
+ text: $draft,
+ axis: .vertical
+ )
.lineLimit(1...5)
.padding(.horizontal, 16)
.padding(.vertical, 10)
- .background(Capsule().fill(Color.secondary.opacity(0.16)))
+ // Capsule's radius is half the height, so a wrapped field
+ // becomes a fat oval. A fixed radius stays a pill on one
+ // line and a rounded rectangle on several — the iMessage
+ // shape, and the one the other chat apps use.
+ .background(
+ RoundedRectangle(cornerRadius: 20, style: .continuous)
+ .fill(Color.secondary.opacity(0.16))
+ )
.focused($composerFocused)
.submitLabel(.send)
+ // Typing while partials stream in would fight the next
+ // transcript callback, which rewrites the whole field from
+ // the frozen base. `.disabled` would also fade the text,
+ // which makes dictated words look like a placeholder.
+ // Hit-testing off keeps them readable and not editable.
+ .allowsHitTesting(!dictation.isListening && !sendingAttachments)
// Return sends, Shift+Return breaks the line — the shape
// every chat app has. `.ignored` hands the keypress back to
// the text field, which is what inserts the newline; there is
@@ -279,23 +508,82 @@ struct ChatView: View {
// key is a send — which is what `.submitLabel(.send)` promises
.onSubmit(submit)
- Button {
- submit()
- } label: {
- Image(systemName: "arrow.up")
- .font(.system(size: 16, weight: .bold))
- .foregroundStyle(Color(uiColor: .systemBackground))
- .frame(width: 36, height: 36)
- .background(
- Circle().fill(canSend ? Color.primary : Color.secondary.opacity(0.35))
- )
+ // The mic stays put. Hiding it once text arrives is the
+ // desktop pattern, where Escape stops listening and the
+ // toolbar only has room for one action. A phone has
+ // neither: this is how you stop, and how you add another
+ // sentence by voice after the first one.
+ Button {
+ dictation.toggle(capturing: draft)
+ } label: {
+ Image(systemName: dictation.isListening ? "mic.fill" : "mic")
+ .font(.system(size: 16, weight: .semibold))
+ .foregroundStyle(dictation.isListening ? Color.red : Color.primary)
+ .frame(width: 36, height: 36)
+ .background(
+ Circle().fill(
+ dictation.isListening
+ ? Color.red.opacity(0.2)
+ : Color.secondary.opacity(0.16)
+ )
+ )
+ .symbolEffect(.pulse, isActive: dictation.isListening)
+ }
+ .accessibilityLabel(dictation.isListening ? "Stop dictation" : "Start dictation")
+ .disabled(sendingAttachments)
+
+ if canSend {
+ Button {
+ submit()
+ } label: {
+ Image(systemName: "arrow.up")
+ .font(.system(size: 16, weight: .bold))
+ .foregroundStyle(Color(uiColor: .systemBackground))
+ .frame(width: 36, height: 36)
+ .background(Circle().fill(Color.primary))
+ }
+ .accessibilityLabel("Send message")
+ .animation(.easeOut(duration: 0.15), value: canSend)
+ }
}
- .disabled(!canSend)
- .animation(.easeOut(duration: 0.15), value: canSend)
}
.padding(.horizontal, 14)
.padding(.vertical, 10)
.background(.bar)
+ .photosPicker(
+ isPresented: $pickingPhotos,
+ selection: $photoItems,
+ maxSelectionCount: PendingMedia.maxCount,
+ matching: .images
+ )
+ .onChange(of: photoItems) { _, items in
+ guard !items.isEmpty, !sendingAttachments else { return }
+ Task { await consumePhotos(items) }
+ }
+ .fullScreenCover(isPresented: $pickingCamera) {
+ CameraPicker(
+ onImage: { image in
+ pickingCamera = false
+ guard !sendingAttachments else { return }
+ addPhoto(image)
+ },
+ onCancel: { pickingCamera = false }
+ )
+ .ignoresSafeArea()
+ }
+ .fileImporter(
+ isPresented: $pickingFiles,
+ allowedContentTypes: [.item],
+ allowsMultipleSelection: true
+ ) { result in
+ guard !sendingAttachments else { return }
+ switch result {
+ case let .success(urls):
+ consumeFiles(urls)
+ case .failure:
+ attachError = "Couldn't open that file."
+ }
+ }
}
}
@@ -428,12 +716,14 @@ private struct ActivityShareSheet: UIViewControllerRepresentable {
struct TextBubble: View {
let message: Message
+ @EnvironmentObject private var session: Session
var body: some View {
let mine = message.role == .user
+ let shown = mine ? Attachment.display(message.text ?? "") : nil
HStack {
if mine { Spacer(minLength: 44) }
- VStack(alignment: .leading, spacing: 4) {
+ VStack(alignment: .leading, spacing: 8) {
// rooms attribute each line to the member who said it
if let from = message.from {
Text(from.name)
@@ -443,12 +733,17 @@ struct TextBubble: View {
// Bots get markdown, you do not — the same split the desktop
// makes. Markdown you did not intend is worse than markdown
// you did: a message about `**` should show the asterisks.
- if mine {
- Text(message.text ?? "")
- .font(.system(size: 17))
- .foregroundStyle(Color.primary)
- .textSelection(.enabled)
- .fixedSize(horizontal: false, vertical: true)
+ if mine, let shown {
+ if !shown.caption.isEmpty {
+ Text(shown.caption)
+ .font(.system(size: 17))
+ .foregroundStyle(Color.primary)
+ .textSelection(.enabled)
+ .fixedSize(horizontal: false, vertical: true)
+ }
+ ForEach(shown.files, id: \.path) { file in
+ InboxAttachmentView(file: file, cached: session.preview(for: file.path))
+ }
} else {
MarkdownText(source: message.text ?? "")
.foregroundStyle(Color.primary)
diff --git a/ios/App/ComposerAttach.swift b/ios/App/ComposerAttach.swift
new file mode 100644
index 0000000000..773bbbad7a
--- /dev/null
+++ b/ios/App/ComposerAttach.swift
@@ -0,0 +1,288 @@
+// Files waiting to go with the next message: a photo, a camera shot, or
+// something from Files. The sidecar writes each onto the computer on send;
+// until then they live only on the phone.
+import SwiftUI
+import UIKit
+import ImageIO
+import CompanionCore
+
+struct PendingAttachment: Identifiable {
+ let id: UUID
+ var name: String
+ var data: Data
+ var preview: UIImage?
+ /// Set once the sidecar has the bytes. A failed send can retry without
+ /// writing the file twice.
+ var host: Attachment.File?
+
+ init(
+ id: UUID = UUID(),
+ name: String,
+ data: Data,
+ preview: UIImage? = nil,
+ host: Attachment.File? = nil
+ ) {
+ self.id = id
+ self.name = name
+ self.data = data
+ self.preview = preview
+ self.host = host
+ }
+}
+
+enum PendingMedia {
+ /// Matches `MAX_INBOX_BYTES` in companion/src/inbox.ts. Keep them in
+ /// step: the phone should refuse before the sidecar does.
+ static let maxBytes = 8 * 1024 * 1024
+ /// Same ceiling as the photo picker. Files are held in memory until send.
+ static let maxCount = 8
+ /// Longest side of a chip / bubble thumbnail. Full JPEG bytes still go
+ /// to the sidecar; only the pixels we keep around for UI are scaled.
+ static let previewMaxPixelSize = 480
+
+ static func thumbnail(from image: UIImage) -> UIImage {
+ let longest = max(image.size.width * image.scale, image.size.height * image.scale)
+ let maxPx = CGFloat(previewMaxPixelSize)
+ guard longest > maxPx, longest > 0 else { return image }
+ let scale = maxPx / longest
+ let size = CGSize(width: image.size.width * scale, height: image.size.height * scale)
+ let format = UIGraphicsImageRendererFormat.default()
+ format.scale = 1
+ return UIGraphicsImageRenderer(size: size, format: format).image { _ in
+ image.draw(in: CGRect(origin: .zero, size: size))
+ }
+ }
+
+ static func thumbnail(from data: Data) -> UIImage? {
+ let sourceOptions: [CFString: Any] = [kCGImageSourceShouldCache: false]
+ guard let source = CGImageSourceCreateWithData(data as CFData, sourceOptions as CFDictionary) else {
+ return UIImage(data: data).map(thumbnail(from:))
+ }
+ let options: [CFString: Any] = [
+ kCGImageSourceCreateThumbnailFromImageAlways: true,
+ kCGImageSourceCreateThumbnailWithTransform: true,
+ kCGImageSourceShouldCacheImmediately: true,
+ kCGImageSourceThumbnailMaxPixelSize: previewMaxPixelSize,
+ ]
+ guard let image = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else {
+ return UIImage(data: data).map(thumbnail(from:))
+ }
+ return UIImage(cgImage: image)
+ }
+
+ static func jpegData(from image: UIImage) -> Data? {
+ let longest = max(image.size.width, image.size.height)
+ let scale = longest > 2048 ? 2048 / longest : 1
+ let size = CGSize(width: image.size.width * scale, height: image.size.height * scale)
+ let renderer = UIGraphicsImageRenderer(size: size)
+ let scaled = renderer.image { _ in image.draw(in: CGRect(origin: .zero, size: size)) }
+ for quality in [CGFloat(0.85), 0.7, 0.55, 0.4] {
+ if let data = scaled.jpegData(compressionQuality: quality), data.count <= maxBytes {
+ return data
+ }
+ }
+ guard let data = scaled.jpegData(compressionQuality: 0.3), data.count <= maxBytes else {
+ return nil
+ }
+ return data
+ }
+
+ static func jpegAttachment(from image: UIImage, name: String) -> PendingAttachment? {
+ guard let data = jpegData(from: image) else { return nil }
+ return PendingAttachment(name: name, data: data, preview: thumbnail(from: image))
+ }
+}
+
+struct ComposerAttachMenu: View {
+ var enabled: Bool
+ var onAttachImage: () -> Void
+ var onTakePhoto: () -> Void
+ var onChooseFile: () -> Void
+
+ var body: some View {
+ Menu {
+ Button(action: onAttachImage) {
+ Label("Attach Image", systemImage: "photo")
+ }
+ Button(action: onTakePhoto) {
+ Label("Take Photo", systemImage: "camera")
+ }
+ Button(action: onChooseFile) {
+ Label("Choose File", systemImage: "folder")
+ }
+ } label: {
+ Image(systemName: "plus")
+ .font(.system(size: 16, weight: .semibold))
+ .foregroundStyle(Color.primary)
+ .frame(width: 36, height: 36)
+ .background(Circle().fill(Color.secondary.opacity(0.16)))
+ }
+ .disabled(!enabled)
+ .accessibilityLabel("Attach")
+ }
+}
+
+struct ComposerAttachBar: View {
+ @Binding var items: [PendingAttachment]
+ var error: String?
+ var enabled: Bool = true
+
+ var body: some View {
+ VStack(alignment: .leading, spacing: 8) {
+ if let error, !error.isEmpty {
+ Text(error)
+ .font(.system(size: 13))
+ .foregroundStyle(.orange)
+ .padding(.horizontal, 4)
+ }
+ if !items.isEmpty {
+ ScrollView(.horizontal, showsIndicators: false) {
+ HStack(spacing: 8) {
+ ForEach(items) { item in
+ chip(item)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private func chip(_ item: PendingAttachment) -> some View {
+ HStack(spacing: 6) {
+ if let preview = item.preview {
+ Image(uiImage: preview)
+ .resizable()
+ .scaledToFill()
+ .frame(width: 28, height: 28)
+ .clipShape(RoundedRectangle(cornerRadius: 6, style: .continuous))
+ } else {
+ Image(systemName: "doc")
+ .font(.system(size: 13, weight: .semibold))
+ .frame(width: 28, height: 28)
+ }
+ Text(item.name)
+ .font(.system(size: 13))
+ .lineLimit(1)
+ Button {
+ items.removeAll { $0.id == item.id }
+ } label: {
+ Image(systemName: "xmark")
+ .font(.system(size: 11, weight: .bold))
+ .foregroundStyle(Color.secondary)
+ }
+ .disabled(!enabled)
+ .accessibilityLabel("Remove \(item.name)")
+ }
+ .padding(.leading, 4)
+ .padding(.trailing, 8)
+ .padding(.vertical, 4)
+ .background(
+ RoundedRectangle(cornerRadius: 12, style: .continuous)
+ .fill(Color.secondary.opacity(0.16))
+ )
+ }
+}
+
+/// JPEG bytes for a photo this phone already uploaded, keyed by the inbox
+/// filename. The bubble should not have to round-trip to the Mac to show
+/// something it held a moment ago.
+enum InboxCache {
+ private static let folder: URL = {
+ let url = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0]
+ .appendingPathComponent("OpenMausInbox", isDirectory: true)
+ try? FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)
+ return url
+ }()
+
+ /// Inbox names are basename-only and already sanitised on the Mac.
+ /// Anything else is not one of ours — including `..`, which
+ /// `appendingPathComponent` would walk out of this folder.
+ private static func storedName(from hostPath: String) -> String? {
+ let name = URL(fileURLWithPath: hostPath).lastPathComponent
+ guard name.range(
+ of: "^[A-Za-z0-9][A-Za-z0-9._-]{0,120}$",
+ options: .regularExpression
+ ) != nil else { return nil }
+ return name
+ }
+
+ static func save(_ data: Data, hostPath: String) {
+ guard let name = storedName(from: hostPath) else { return }
+ try? data.write(to: folder.appendingPathComponent(name), options: .atomic)
+ }
+
+ static func load(hostPath: String) -> Data? {
+ guard let name = storedName(from: hostPath) else { return nil }
+ return try? Data(contentsOf: folder.appendingPathComponent(name))
+ }
+}
+
+/// A file that already went with a message: the photo if we can show it,
+/// otherwise a chip with the name. The sidecar still has the bytes.
+struct InboxAttachmentView: View {
+ let file: Attachment.File
+ var cached: UIImage?
+ @EnvironmentObject private var session: Session
+ @State private var loaded: UIImage?
+ @State private var tried = false
+
+ private var pixels: UIImage? { cached ?? loaded }
+
+ var body: some View {
+ Group {
+ if file.isImage, let pixels {
+ Image(uiImage: pixels)
+ .resizable()
+ .scaledToFit()
+ .frame(maxHeight: 240)
+ .clipShape(RoundedRectangle(cornerRadius: 14, style: .continuous))
+ } else if file.isImage, !tried {
+ RoundedRectangle(cornerRadius: 14, style: .continuous)
+ .fill(Color.secondary.opacity(0.16))
+ .frame(height: 160)
+ .overlay { ProgressView() }
+ } else if file.isImage {
+ Button {
+ tried = false
+ Task { await load() }
+ } label: {
+ Label(file.displayName, systemImage: "photo")
+ .font(.system(size: 15))
+ .padding(.horizontal, 10)
+ .padding(.vertical, 8)
+ .background(
+ RoundedRectangle(cornerRadius: 10, style: .continuous)
+ .fill(Color.secondary.opacity(0.16))
+ )
+ }
+ .buttonStyle(.plain)
+ } else {
+ Label(file.displayName, systemImage: "doc")
+ .font(.system(size: 15))
+ .padding(.horizontal, 10)
+ .padding(.vertical, 8)
+ .background(
+ RoundedRectangle(cornerRadius: 10, style: .continuous)
+ .fill(Color.secondary.opacity(0.16))
+ )
+ }
+ }
+ .accessibilityLabel(file.displayName)
+ .task(id: file.path) { await load() }
+ }
+
+ private func load() async {
+ guard file.isImage, pixels == nil else { return }
+ if let data = InboxCache.load(hostPath: file.path), let decoded = PendingMedia.thumbnail(from: data) {
+ loaded = decoded
+ return
+ }
+ let name = URL(fileURLWithPath: file.path).lastPathComponent
+ if let data = await session.inboxFile(named: name), let decoded = PendingMedia.thumbnail(from: data) {
+ InboxCache.save(data, hostPath: file.path)
+ loaded = decoded
+ return
+ }
+ tried = true
+ }
+}
diff --git a/ios/App/MarkdownText.swift b/ios/App/MarkdownText.swift
index 2c6341acbc..98c449ec82 100644
--- a/ios/App/MarkdownText.swift
+++ b/ios/App/MarkdownText.swift
@@ -87,9 +87,82 @@ struct MarkdownText: View {
case .rule:
Divider().padding(.vertical, 2)
+
+ case let .table(headers, alignments, rows):
+ pipeTable(headers: headers, alignments: alignments, rows: rows, tail: tail)
}
}
+ /// Wide tables scroll sideways the way fenced code does. A panel
+ /// directory cannot wrap cell-by-cell without becoming the pipe soup
+ /// this exists to replace.
+ private func pipeTable(
+ headers: [String],
+ alignments: [MarkdownTableAlignment],
+ rows: [[String]],
+ tail: Bool
+ ) -> some View {
+ VStack(alignment: .leading, spacing: 0) {
+ ScrollView(.horizontal, showsIndicators: false) {
+ Grid(alignment: .leading, horizontalSpacing: 0, verticalSpacing: 0) {
+ GridRow {
+ ForEach(headers.indices, id: \.self) { column in
+ tableCell(
+ headers[column],
+ header: true,
+ alignment: alignments[column]
+ )
+ }
+ }
+ ForEach(rows.indices, id: \.self) { row in
+ GridRow {
+ ForEach(headers.indices, id: \.self) { column in
+ tableCell(
+ rows[row][column],
+ header: false,
+ alignment: alignments[column]
+ )
+ }
+ }
+ }
+ }
+ .fixedSize(horizontal: true, vertical: true)
+ }
+ if tail {
+ caretText(true)
+ .font(.system(size: 14))
+ .padding(.top, 4)
+ }
+ }
+ }
+
+ private func tableCell(_ text: String, header: Bool, alignment: MarkdownTableAlignment) -> some View {
+ let frame: Alignment
+ let wrap: TextAlignment
+ switch alignment {
+ case .left:
+ frame = .leading
+ wrap = .leading
+ case .center:
+ frame = .center
+ wrap = .center
+ case .right:
+ frame = .trailing
+ wrap = .trailing
+ }
+ return inline(text)
+ .font(.system(size: 14, weight: header ? .semibold : .regular))
+ .multilineTextAlignment(wrap)
+ .frame(minWidth: 28, alignment: frame)
+ .padding(.horizontal, 8)
+ .padding(.vertical, 6)
+ .overlay(alignment: .bottom) {
+ Rectangle()
+ .fill(Color.secondary.opacity(header ? 0.35 : 0.16))
+ .frame(height: 1)
+ }
+ }
+
private func marker(_ symbol: String, indent: Int, text: String, tail: Bool) -> some View {
HStack(alignment: .firstTextBaseline, spacing: 6) {
Text(symbol)
diff --git a/ios/App/Session.swift b/ios/App/Session.swift
index 0de8b8954c..573a67d2b1 100644
--- a/ios/App/Session.swift
+++ b/ios/App/Session.swift
@@ -10,9 +10,9 @@
import Foundation
import OSLog
import SwiftUI
+import UIKit
import CompanionCore
import UserNotifications
-import UIKit
/// Stream lifecycle, in Console.app and the Xcode console. A companion that
/// is silently not connected looks exactly like one with nothing to say, so
@@ -55,6 +55,9 @@ final class Session: ObservableObject {
/// can finish after its replacement starts; its cleanup must not clear
/// the replacement's handle.
private var streamGeneration = 0
+ /// Bumped when the paired computer changes so an in-flight upload cannot
+ /// be sent through a later client.
+ private var clientGeneration = 0
private var reconnectDelay: UInt64 = 0
/// How many computer panels are open. A count rather than a flag: the
/// panel can be pushed twice in a navigation stack, and the last one to
@@ -149,11 +152,13 @@ final class Session: ObservableObject {
try Keychain.save(paired.token, for: stored.id)
UserDefaults.standard.set(try? JSONEncoder().encode(stored), forKey: Self.connectionKey)
+ clientGeneration += 1
self.connection = stored
self.token = paired.token
self.rotation = CandidateRotation(hosts: stored.orderedHosts)
self.client = CompanionClient(connection: stored, token: paired.token)
self.state = CompanionState()
+ forgetAttachmentPreviews()
// A fresh pairing settles any restore that was still waiting on the
// keychain — the token is in hand, so there is nothing left to retry.
restorePending = false
@@ -177,6 +182,7 @@ final class Session: ObservableObject {
}
func signOut() {
+ clientGeneration += 1
streamTask?.cancel()
streamTask = nil
restorePending = false
@@ -187,6 +193,7 @@ final class Session: ObservableObject {
token = nil
rotation = CandidateRotation(hosts: [])
state = CompanionState()
+ forgetAttachmentPreviews()
NotificationCoordinator.shared.setBadge(0)
status = .unpaired
}
@@ -401,12 +408,42 @@ final class Session: ObservableObject {
// the source of truth, and a phone that draws its own version of events
// is a phone that disagrees with the laptop.
- func send(_ text: String, to chat: Chat) async {
- await perform {
+ func send(_ text: String, to chat: Chat) async -> Bool {
+ actionError = nil
+ let generation = clientGeneration
+ guard let client else { return false }
+ do {
switch chat {
- case let .bot(bot): try await $0.send(text: text, toBot: bot.id)
- case let .room(room): try await $0.send(text: text, toRoom: room.id)
+ case let .bot(bot): try await client.send(text: text, toBot: bot.id)
+ case let .room(room): try await client.send(text: text, toRoom: room.id)
}
+ guard generation == clientGeneration else { return false }
+ return true
+ } catch let error as APIError where error.isUnauthorized {
+ status = .unauthorized
+ return false
+ } catch {
+ actionError = error.localizedDescription
+ return false
+ }
+ }
+
+ /// Write a phone file onto the computer. Returns the host path to fold
+ /// into the next message; the harness never sees the bytes.
+ func upload(_ data: Data, filename: String) async -> InboxFile? {
+ actionError = nil
+ let generation = clientGeneration
+ guard let client else { return nil }
+ do {
+ let stored = try await client.upload(data: data, filename: filename)
+ guard generation == clientGeneration else { return nil }
+ return stored
+ } catch let error as APIError where error.isUnauthorized {
+ status = .unauthorized
+ return nil
+ } catch {
+ actionError = error.localizedDescription
+ return nil
}
}
@@ -494,6 +531,18 @@ final class Session: ObservableObject {
try? await client?.image(threadId: threadId, messageId: messageId)
}
+ func inboxFile(named name: String) async -> Data? {
+ guard let client else { return nil }
+ do {
+ return try await client.inboxFile(named: name)
+ } catch let error as APIError where error.isUnauthorized {
+ status = .unauthorized
+ return nil
+ } catch {
+ return nil
+ }
+ }
+
func search(_ query: String) async -> [SearchHit] {
let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.count >= 2, let client else { return [] }
@@ -598,6 +647,32 @@ final class Session: ObservableObject {
}
}
+ /// Thumbnails of files this session just sent, keyed by the host path
+ /// in the message. The bubble uses them so a photo appears before the
+ /// sidecar round-trip, and after a restart the GET above fills in.
+ private static let maxAttachmentPreviews = 24
+ private var attachmentPreviews: [String: UIImage] = [:]
+ private var attachmentPreviewOrder: [String] = []
+
+ func rememberPreview(_ image: UIImage, for path: String) {
+ attachmentPreviews[path] = PendingMedia.thumbnail(from: image)
+ attachmentPreviewOrder.removeAll { $0 == path }
+ attachmentPreviewOrder.append(path)
+ while attachmentPreviewOrder.count > Self.maxAttachmentPreviews {
+ let old = attachmentPreviewOrder.removeFirst()
+ attachmentPreviews.removeValue(forKey: old)
+ }
+ }
+
+ func preview(for path: String) -> UIImage? {
+ attachmentPreviews[path]
+ }
+
+ private func forgetAttachmentPreviews() {
+ attachmentPreviews.removeAll()
+ attachmentPreviewOrder.removeAll()
+ }
+
func refreshNotificationAuthorization() async {
notificationAuthorization = await NotificationCoordinator.shared.authorizationStatus()
}
@@ -765,11 +840,23 @@ extension CompanionState {
private static func preview(of last: Message?) -> String {
guard let last else { return "" }
switch last.kind {
- case .text: return last.text ?? ""
+ case .text:
+ return Self.previewText(last.text ?? "")
case .options: return last.card?.isPending == true ? "Waiting on you" : (last.card?.title ?? "")
case .activity: return last.tool?.name ?? ""
case .screen: return "Screenshot"
case .unknown: return last.text ?? ""
}
}
+
+ /// Hide `` tags in the roster. The path is for the agent.
+ private static func previewText(_ text: String) -> String {
+ let shown = Attachment.display(text)
+ if shown.files.isEmpty { return shown.caption }
+ if shown.caption.isEmpty {
+ if shown.files.count == 1 { return shown.files[0].displayName }
+ return "\(shown.files.count) files"
+ }
+ return shown.caption
+ }
}
diff --git a/ios/App/SpeechDictation.swift b/ios/App/SpeechDictation.swift
new file mode 100644
index 0000000000..8a483475cf
--- /dev/null
+++ b/ios/App/SpeechDictation.swift
@@ -0,0 +1,254 @@
+// On-device dictation for the composer.
+//
+// Same engine as the desktop helper (`electron/resources/speech-helper.swift`):
+// `SFSpeechRecognizer` on an `AVAudioEngine` tap, partials streamed into the
+// text field, press to stop. Composer mode, not call mode — there is no
+// silence endpointing. The phone is better at this than the Mac was: the
+// recognizer is in the same process as the field, so there is no helper
+// binary, no TCC bundle dance, and no `open -W`.
+//
+// On-device when the recognizer supports it, so talking to a bot does not
+// become talking to Apple's servers. Locales come from `Dictation.localeCandidates`
+// rather than a hardcoded en-US, for the same reason the desktop helper
+// stopped hardcoding one.
+//
+// Lives in the app target on purpose. CompanionCore is Foundation-only so
+// `swift test` can run without a simulator; Speech and AVAudioEngine are
+// the opposite of that.
+import AVFoundation
+import Speech
+import CompanionCore
+
+@MainActor
+final class SpeechDictation: ObservableObject {
+ @Published private(set) var isListening = false
+ @Published private(set) var transcript = ""
+ @Published private(set) var error: String?
+
+ /// Composer text captured when listening started. Frozen for the
+ /// session so each partial replaces the last rather than stacking.
+ /// ChatView reads this from `onChange(of: transcript)` and must not
+ /// substitute the live draft.
+ private(set) var base = ""
+
+ private var recognizer: SFSpeechRecognizer?
+ private var audioEngine: AVAudioEngine?
+ private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
+ private var recognitionTask: SFSpeechRecognitionTask?
+ private var tapInstalled = false
+ private var stopping = false
+ /// True from `start` until capture is running or the attempt fails.
+ /// `isListening` is only true once the engine is up, so without this
+ /// a second tap during the permission prompts would call `start`
+ /// again instead of cancelling.
+ private var starting = false
+ /// Bumped on every start/stop so an authorization that finishes after
+ /// the user already cancelled cannot open the mic.
+ private var generation = 0
+ private var startTask: Task?
+
+ func toggle(capturing base: String) {
+ if isListening || starting {
+ stop()
+ } else {
+ start(base: base)
+ }
+ }
+
+ private func start(base: String) {
+ guard !isListening, !starting else { return }
+ error = nil
+ self.base = base.trimmingCharacters(in: .whitespacesAndNewlines)
+ transcript = ""
+ starting = true
+ generation += 1
+ let gen = generation
+ startTask = Task { await actuallyStart(generation: gen) }
+ }
+
+ func stop() {
+ startTask?.cancel()
+ startTask = nil
+ generation += 1
+ starting = false
+ stopping = true
+ isListening = false
+ teardown()
+ }
+
+ // MARK: - Authorization
+
+ private func actuallyStart(generation gen: Int) async {
+ let speech = await requestSpeechAuthorization()
+ guard gen == generation, !Task.isCancelled else {
+ starting = false
+ return
+ }
+ guard speech == .authorized else {
+ starting = false
+ error = Self.speechDeniedMessage
+ return
+ }
+
+ let mic = await AVAudioApplication.requestRecordPermission()
+ guard gen == generation, !Task.isCancelled else {
+ starting = false
+ return
+ }
+ guard mic else {
+ starting = false
+ error = Self.micDeniedMessage
+ return
+ }
+
+ do {
+ try beginCapture(generation: gen)
+ starting = false
+ } catch CaptureError.noRecognizer {
+ starting = false
+ error = "Dictation isn't available for this language."
+ teardown()
+ } catch {
+ starting = false
+ self.error = "Couldn't start the microphone."
+ teardown()
+ }
+ }
+
+ private func requestSpeechAuthorization() async -> SFSpeechRecognizerAuthorizationStatus {
+ await withCheckedContinuation { continuation in
+ SFSpeechRecognizer.requestAuthorization { status in
+ continuation.resume(returning: status)
+ }
+ }
+ }
+
+ // MARK: - Capture
+
+ private func beginCapture(generation gen: Int) throws {
+ let recognizer = Dictation.localeCandidates()
+ .compactMap { SFSpeechRecognizer(locale: $0) }
+ .first { $0.isAvailable }
+ guard let recognizer else {
+ throw CaptureError.noRecognizer
+ }
+ self.recognizer = recognizer
+
+ let session = AVAudioSession.sharedInstance()
+ // `.record` rather than `.playAndRecord`: this is composer
+ // dictation, not a call, and holding the playback route would
+ // duck whatever else is on the phone for no reason.
+ try session.setCategory(.record, mode: .measurement)
+ try session.setActive(true, options: .notifyOthersOnDeactivation)
+
+ let engine = AVAudioEngine()
+ let request = SFSpeechAudioBufferRecognitionRequest()
+ request.shouldReportPartialResults = true
+ // The desktop helper does not set this (it is a CLI talking to an
+ // older Speech.framework), but a chat message is better with the
+ // commas the recognizer already knows about.
+ request.addsPunctuation = true
+ request.taskHint = .dictation
+ if recognizer.supportsOnDeviceRecognition {
+ request.requiresOnDeviceRecognition = true
+ }
+
+ // Keep the engine on self before start() so a throw still has
+ // something for teardown to remove the tap from. A local engine
+ // that fails to start would leave tapInstalled true and the next
+ // teardown would removeTap on a new engine that has none — which
+ // is an exception, not a no-op.
+ audioEngine = engine
+ recognitionRequest = request
+
+ // The tap format is only valid after the session is active.
+ // Installing against a 0-channel format is the usual "it works
+ // in the sample and fails here" failure.
+ let input = engine.inputNode
+ let format = input.outputFormat(forBus: 0)
+ guard format.channelCount > 0 else {
+ throw CaptureError.silentInput
+ }
+ input.installTap(onBus: 0, bufferSize: 1024, format: format) { buffer, _ in
+ request.append(buffer)
+ }
+ tapInstalled = true
+ engine.prepare()
+ try engine.start()
+
+ stopping = false
+ isListening = true
+
+ recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, recognitionError in
+ Task { @MainActor in
+ self?.handle(
+ result: result,
+ recognitionError: recognitionError,
+ generation: gen
+ )
+ }
+ }
+ }
+
+ private func handle(
+ result: SFSpeechRecognitionResult?,
+ recognitionError: Error?,
+ generation gen: Int
+ ) {
+ // A cancelled task can still deliver a partial or a 209 after the
+ // next session has already started. `isListening` is true then too,
+ // so generation is what keeps this callback from rewriting the
+ // new draft or stopping the new capture.
+ guard gen == generation, !stopping, isListening else { return }
+ if let result {
+ transcript = result.bestTranscription.formattedString
+ // Composer dictation does not wait for isFinal — the last
+ // partial is what you send. If the recognizer finalizes on
+ // its own (rare without endAudio), just stop listening.
+ if result.isFinal {
+ stop()
+ return
+ }
+ }
+ guard let recognitionError else { return }
+ let ns = recognitionError as NSError
+ // 209/216 are the cancellation codes Speech uses when we tear
+ // the task down ourselves. Surfacing those as "Couldn't
+ // transcribe that" is how a tap-to-stop looks like a failure.
+ if ns.domain == "kLSRErrorDomain", ns.code == 209 || ns.code == 216 {
+ stop()
+ return
+ }
+ self.error = "Couldn't transcribe that."
+ stop()
+ }
+
+ private func teardown() {
+ // Drop the tap before ending the request: a buffer that arrives
+ // after endAudio() can fail the task instead of being ignored.
+ if let engine = audioEngine {
+ if tapInstalled {
+ engine.inputNode.removeTap(onBus: 0)
+ tapInstalled = false
+ }
+ if engine.isRunning { engine.stop() }
+ }
+ recognitionTask?.cancel()
+ recognitionTask = nil
+ recognitionRequest?.endAudio()
+ recognitionRequest = nil
+ audioEngine = nil
+ recognizer = nil
+ try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
+ }
+
+ private enum CaptureError: Error {
+ case silentInput
+ case noRecognizer
+ }
+
+ static let speechDeniedMessage =
+ "Dictation needs Speech Recognition access. Enable it in Settings → OpenMausMobile."
+ static let micDeniedMessage =
+ "Dictation needs Microphone access. Enable it in Settings → OpenMausMobile."
+}
diff --git a/ios/README.md b/ios/README.md
index ac35902ed9..1ba1de7e16 100644
--- a/ios/README.md
+++ b/ios/README.md
@@ -34,7 +34,7 @@ real `URLSession` tests:
## Layout
-```
+```text
ios/
Package.swift CompanionCore + its tests
project.yml XcodeGen spec for the app target
@@ -44,11 +44,15 @@ ios/
SSE.swift line parser + URLSession event stream
Client.swift every call the phone is allowed to make
Store.swift the fold: frames → state
+ Dictation.swift composer text + transcript join
+ Attachments.swift composer text + host file paths
Tests/CompanionCoreTests/
Fixtures/ captured from a real server — do not hand-edit
DecodingTests.swift the contract with the harness
SSETests.swift the parser, which is where this goes wrong
StoreTests.swift the fold
+ DictationTests.swift partials replace, they do not stack
+ AttachmentTests.swift tagged paths, escaped attributes
App/ SwiftUI, and everything that needs a device
CompanionApp.swift entry; owns when the stream lives and dies
Session.swift connection, lifecycle, actions
@@ -59,6 +63,10 @@ ios/
PairingScanner.swift native QR camera, permission and recovery UI
ChatListView.swift roster, with "waiting on you" pulled to the top
ChatView.swift transcript, approval cards, composer
+ SpeechDictation.swift SFSpeechRecognizer, press-to-stop
+ ComposerAttach.swift pending files, plus menu, chips
+ CameraPicker.swift take a photo for the composer
+ TaskManagerView.swift create, switch, rename, and delete tasks
ComputerView.swift opt-in live view of a bot's computer
MarkdownText.swift the supported Markdown presentation layer
SettingsView.swift status, and unpair
@@ -80,17 +88,23 @@ brew install xcodegen
cd ios && xcodegen generate && open OpenMausCompanion.xcodeproj
```
-**Re-run `xcodegen generate` after pulling any change that adds a file to
-`App/`.** The spec says `sources: App`, but XcodeGen resolves that to explicit
-file references when it generates, so a new file is simply absent from the
-target until you regenerate — and the build fails with `Cannot find 'X' in
-scope`, which reads like a code error and is not one.
+**Re-run `xcodegen generate` once after pulling a `project.yml` change.**
+`App/` is an Xcode 16 synced folder, so new Swift files in it show up on
+the next build without regenerating. Quit Xcode before `git pull`, then
+open the project again — pulling while it is open is what produces
+"Build input files cannot be found" for `CameraPicker.swift` /
+`SpeechDictation.swift`.
If you'd rather not install XcodeGen, make an iOS App target by hand, add the
`App/` folder and the local `CompanionCore` package, and copy the Info.plist
keys out of `project.yml` — `NSLocalNetworkUsageDescription` and
-`NSBonjourServices` especially. Without them `NWBrowser` returns no results at
-all, *silently*, which looks exactly like "no computers on this network".
+`NSBonjourServices` especially, plus `NSMicrophoneUsageDescription` and
+`NSSpeechRecognitionUsageDescription` for the composer mic, and
+`NSCameraUsageDescription` for Take Photo. Without the Bonjour pair,
+`NWBrowser` returns no results at all, *silently*, which looks exactly like
+"no computers on this network". Without the speech pair, the first tap on
+the mic crashes rather than prompting. Without the camera string, Take Photo
+does the same.
## Regenerating the fixtures
@@ -119,6 +133,7 @@ here by simply not having the methods:
| **Answer approvals and questions** | Drive the Local VM or this computer |
| Interrupt a bot, mark chats read | Reach `/api/internal/*` |
| Fetch screen images on demand | Load the packaged desktop UI |
+| Attach a photo or file (`POST /api/inbox`) | Write bytes into the harness |
| Open an explicitly enabled cloud desktop | Provision, sleep or run shell commands on cloud computers |
Marking a chat read and remembering an approval use purpose-built server
@@ -166,20 +181,26 @@ the host computer remain unreachable through the companion.
`.ignored` for the shifted case hands the keypress back to the text field,
which is the only thing that can insert the newline once Return is claimed.
Software keyboards have no Shift+Return, so there `.onSubmit` sends.
-- **No affordance without a feature behind it.** The reference design this was
- modelled on has a composer mic; there is no dictation here, so it is not
- drawn. Search covers the SQLite transcript store and opens the exact task,
- branch, and message; the roster's "+" creates the same basic bot the desktop
- endpoint creates, then opens it.
+- **Composer dictation is the mic.** Tap to talk, tap to stop, then send or
+ edit — the same press-to-stop shape as the desktop composer, on-device
+ `SFSpeechRecognizer` when the phone supports it. The mic stays next to
+ send so you can add another sentence by voice, and so you can stop
+ without an Escape key. Search covers the SQLite transcript store and
+ opens the exact task, branch, and message; the roster's "+" creates the
+ same basic bot the desktop endpoint creates, then opens it.
+- **Composer attach is a photo or a file.** Plus menu: library, camera, or
+ Files. The sidecar writes the bytes onto this computer (`POST /api/inbox`)
+ and the message the bot receives is the same ``
+ tag the desktop composer already sends. The harness never sees the bytes.
## Not in this version
The live connection is foreground-only. Notification frames produce native
banners, sounds, time-sensitive approval alerts, and an app badge while connected;
the resume cursor replays alerts missed during a short background pause. There is
-no APNs delivery after the app is terminated, no voice/call mode, and no hosted relay.
-Task management, SQLite transcript search,
-transcript sharing, reactions, and edit/version controls use narrow companion
-routes and the computer remains the source of truth. Tailscale is supported
-through manual MagicDNS entry; it is not a dependency and OpenMausBot does not
-operate a cloud copy of local data.
+no APNs delivery after the app is terminated, no call mode or spoken replies,
+and no hosted relay. Composer dictation and photo/file attach are in. Task
+management, SQLite transcript search, transcript sharing, reactions, and
+edit/version controls use narrow companion routes and the computer remains the
+source of truth. Tailscale is supported through manual MagicDNS entry; it is
+not a dependency and OpenMausBot does not operate a cloud copy of local data.
diff --git a/ios/Sources/CompanionCore/Attachments.swift b/ios/Sources/CompanionCore/Attachments.swift
new file mode 100644
index 0000000000..8a64732813
--- /dev/null
+++ b/ios/Sources/CompanionCore/Attachments.swift
@@ -0,0 +1,118 @@
+// Composer attachments, the half that has no photo library.
+//
+// The phone picks a file; the sidecar writes it onto this computer; the
+// message the bot receives is the same tagged path the desktop composer
+// already sends (`src/lib/composer-attachments.ts`). Drivers never see
+// bytes from the phone — they open a path that now exists on the host.
+import Foundation
+
+public enum Attachment {
+ public struct File: Equatable, Sendable {
+ public var path: String
+ public var name: String
+ public var size: Int
+
+ public init(path: String, name: String, size: Int) {
+ self.path = path
+ self.name = name
+ self.size = size
+ }
+
+ /// Cheap, from the name. The view uses this to decide thumbnail vs chip.
+ public var isImage: Bool {
+ let ext = URL(fileURLWithPath: name).pathExtension.lowercased()
+ return Self.imageExtensions.contains(ext)
+ }
+
+ /// Inbox files are stored as `timestamp-hex-original.jpg`. Show the
+ /// original in a chip; the prefix is only uniqueness on disk.
+ public var displayName: String {
+ let base = URL(fileURLWithPath: name).lastPathComponent
+ let parts = base.split(separator: "-", maxSplits: 2, omittingEmptySubsequences: false)
+ guard parts.count == 3,
+ parts[0].allSatisfy(\.isNumber),
+ parts[1].count == 8,
+ parts[1].allSatisfy(\.isHexDigit)
+ else { return base }
+ return String(parts[2])
+ }
+
+ private static let imageExtensions: Set = [
+ "jpg", "jpeg", "png", "gif", "heic", "heif", "webp", "tif", "tiff",
+ ]
+ }
+
+ /// What a person should see: the caption they typed, then the files, never
+ /// the host path tag. That tag is for the agent.
+ public struct Display: Equatable, Sendable {
+ public var caption: String
+ public var files: [File]
+ }
+
+ /// Combine typed composer text with host paths for attached files.
+ ///
+ /// Empty text is allowed when there is at least one file — a photo with
+ /// no caption is still a message. Empty everything stays empty so the
+ /// caller can refuse to send.
+ public static func draft(text: String, files: [File]) -> String {
+ var parts: [String] = []
+ let typed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+ if !typed.isEmpty { parts.append(typed) }
+ for file in files {
+ parts.append("")
+ }
+ return parts.joined(separator: "\n\n")
+ }
+
+ /// File paths are untrusted prompt content. Keep them inside the quoted
+ /// attribute even when a filename contains XML characters or line breaks.
+ public static func escapeAttribute(_ value: String) -> String {
+ value
+ .replacingOccurrences(of: "&", with: "&")
+ .replacingOccurrences(of: "\"", with: """)
+ .replacingOccurrences(of: "<", with: "<")
+ .replacingOccurrences(of: ">", with: ">")
+ .replacingOccurrences(of: "\t", with: " ")
+ .replacingOccurrences(of: "\r", with: "
")
+ .replacingOccurrences(of: "\n", with: "
")
+ }
+
+ /// Inverse of `escapeAttribute`. `&` last, so a path that encoded an
+ /// ampersand does not get scanned for a second round of entities.
+ public static func unescapeAttribute(_ value: String) -> String {
+ value
+ .replacingOccurrences(of: "
", with: "\n")
+ .replacingOccurrences(of: "
", with: "\r")
+ .replacingOccurrences(of: " ", with: "\t")
+ .replacingOccurrences(of: """, with: "\"")
+ .replacingOccurrences(of: "<", with: "<")
+ .replacingOccurrences(of: ">", with: ">")
+ .replacingOccurrences(of: "&", with: "&")
+ }
+
+ /// Pull caption and files out of a stored user message. A message with
+ /// no tags is unchanged; tags become files named from the path.
+ public static func display(_ text: String) -> Display {
+ let tag = #//#
+ let matches = text.matches(of: tag)
+ guard !matches.isEmpty else {
+ return Display(caption: text, files: [])
+ }
+
+ var files: [File] = []
+ var parts: [String] = []
+ var cursor = text.startIndex
+ for match in matches {
+ let before = String(text[cursor.. Data {
+ // Same rule the sidecar uses: basename only, no leading dot, no
+ // traversal. The path is interpolated into the URL, so a `../`
+ // here would be a request for a different route entirely.
+ let base = URL(fileURLWithPath: name).lastPathComponent
+ guard base == name,
+ name.range(of: "^[A-Za-z0-9][A-Za-z0-9._-]*$", options: .regularExpression) != nil
+ else { throw APIError.badURL }
+ let encoded = name.addingPercentEncoding(withAllowedCharacters: Self.filenameHeaderAllowed) ?? name
+ var request = try makeRequest("GET", "/api/inbox/\(encoded)")
+ request.timeoutInterval = 60
+ let (data, response) = try await perform(request)
+ try Self.check(response, data)
+ return data
+ }
+
// MARK: - Doing
/// Make a new bot. The harness picks its name, colour and greeting — the
@@ -412,6 +434,25 @@ public struct CompanionClient: Sendable {
try await send(try makeRequest("POST", "/api/groups/\(groupId)/messages", body: ["text": text]))
}
+ /// Put a phone file onto the computer. The sidecar writes the bytes and
+ /// returns a host path; send that path as `` the way the
+ /// desktop composer already does. Not forwarded to the harness.
+ public func upload(data: Data, filename: String) async throws -> InboxFile {
+ var request = try makeRequest("POST", "/api/inbox")
+ // Photos over a tailnet are larger than a chat message. Twenty
+ // seconds is the right timeout for the rest of this client and the
+ // wrong one here.
+ request.timeoutInterval = 60
+ request.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type")
+ // RFC 3986 unreserved only: this value is an HTTP header, so spaces,
+ // quotes and non-ASCII have to leave as percent-escapes. The sidecar
+ // runs them through decodeURIComponent before sanitising the name.
+ let encoded = filename.addingPercentEncoding(withAllowedCharacters: Self.filenameHeaderAllowed) ?? "file"
+ request.setValue(encoded, forHTTPHeaderField: "X-OpenMaus-Filename")
+ request.httpBody = data
+ return try await send(request, as: InboxFile.self)
+ }
+
/// Answer an approval or a question.
///
/// Addressed by thread rather than by bot on purpose: a request raised
diff --git a/ios/Sources/CompanionCore/Dictation.swift b/ios/Sources/CompanionCore/Dictation.swift
new file mode 100644
index 0000000000..140b37acc3
--- /dev/null
+++ b/ios/Sources/CompanionCore/Dictation.swift
@@ -0,0 +1,64 @@
+// Composer dictation, the half that has no microphone.
+//
+// The Speech session lives in the app target — it needs AVFoundation and a
+// device. What lives here is the contract between that session and the text
+// field, because that is where the decisions are and where they can be
+// tested without a phone:
+//
+// - Partials *replace* each other after the text that was already in the
+// composer. They never stack. The desktop helper works the same way
+// (`src/components/Composer.tsx`): the base is frozen when the mic goes
+// on, and every transcript line is `base + " " + spoken`.
+// - The recognizer's locale is the user's language, not a hardcoded
+// English. A French speaker talking to an en-US recognizer gets
+// nonsense, which is how this went wrong on the desktop the first time
+// (`electron/resources/speech-helper.swift`). Same candidate list here.
+import Foundation
+
+public enum Dictation {
+ /// Combine already-typed composer text with the current transcript.
+ ///
+ /// `base` is whatever was in the field when listening started, frozen
+ /// for the session. Pass that every time, not the live draft — passing
+ /// the live draft would append each partial onto the last one.
+ public static func draft(base: String, transcript: String) -> String {
+ let typed = base.trimmingCharacters(in: .whitespacesAndNewlines)
+ let spoken = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
+ if spoken.isEmpty { return typed }
+ if typed.isEmpty { return spoken }
+ return "\(typed) \(spoken)"
+ }
+
+ /// Locales to try, in order. First available recognizer wins.
+ ///
+ /// Preferred languages, then the current locale, then en-US as a last
+ /// resort so a device with no speech support for the user's language
+ /// still has something to attempt rather than failing closed with no
+ /// explanation.
+ public static func localeCandidates(
+ preferredLanguages: [String] = Locale.preferredLanguages,
+ current: Locale = .current
+ ) -> [Locale] {
+ var seen = Set()
+ var result: [Locale] = []
+ func add(_ locale: Locale) {
+ // "en-US" and "en_US" are the same recognizer. Canonicalize so
+ // the fallback does not add a duplicate of a locale we already
+ // tried under a different identifier spelling.
+ let key = canonicalIdentifier(locale)
+ guard seen.insert(key).inserted else { return }
+ result.append(locale)
+ }
+ for language in preferredLanguages {
+ add(Locale(identifier: language))
+ }
+ add(current)
+ add(Locale(identifier: "en-US"))
+ return result
+ }
+
+ /// Lowercased BCP-47 with underscores, so `en-US` and `en_US` collide.
+ public static func canonicalIdentifier(_ locale: Locale) -> String {
+ locale.identifier.lowercased().replacingOccurrences(of: "-", with: "_")
+ }
+}
diff --git a/ios/Sources/CompanionCore/Markdown.swift b/ios/Sources/CompanionCore/Markdown.swift
index 89c8c31310..0bea391415 100644
--- a/ios/Sources/CompanionCore/Markdown.swift
+++ b/ios/Sources/CompanionCore/Markdown.swift
@@ -14,10 +14,16 @@
//
// Deliberately not a full CommonMark implementation. It covers what a model
// actually emits into a chat bubble — paragraphs, lists, headings, fences,
-// quotes, rules — and treats anything it does not recognise as text, which is
-// the failure mode that loses nothing.
+// quotes, rules, GFM tables — and treats anything it does not recognise as
+// text, which is the failure mode that loses nothing.
import Foundation
+public enum MarkdownTableAlignment: Equatable, Sendable {
+ case left
+ case center
+ case right
+}
+
public enum MarkdownBlock: Equatable, Sendable {
case paragraph(String)
/// `indent` is nesting depth, 0 for a top-level item.
@@ -27,6 +33,9 @@ public enum MarkdownBlock: Equatable, Sendable {
/// A fenced block. `language` is whatever followed the opening fence.
case code(language: String?, text: String)
case quote(String)
+ /// A GFM pipe table. Column count comes from the delimiter row, the
+ /// same rule remark-gfm uses on the desktop.
+ case table(headers: [String], alignments: [MarkdownTableAlignment], rows: [[String]])
case rule
}
@@ -84,6 +93,16 @@ public enum Markdown {
continue
}
+ // A GFM table is a header line plus a delimiter row of dashes.
+ // Without the delimiter, a line that happens to contain `|` is
+ // still prose — otherwise "use | as a pipe" becomes a one-cell
+ // table the moment it wraps.
+ if let table = takeTable(trimmed, remaining: &lines) {
+ flushParagraph()
+ blocks.append(table)
+ continue
+ }
+
// --- or *** or ___, three or more, nothing else on the line
if trimmed.count >= 3, "-*_".contains(trimmed.first!),
trimmed.allSatisfy({ $0 == trimmed.first! }) {
@@ -146,4 +165,103 @@ public enum Markdown {
}
return nil
}
+
+ /// Header + delimiter, then body rows until a blank line or a line that
+ /// is not a row. Header and delimiter must have the same cell count —
+ /// GFM's rule, and the one that keeps a streaming `| check | ok |\n| --`
+ /// from becoming a one-column table that drops `ok`.
+ private static func takeTable(_ headerLine: String, remaining: inout ArraySlice) -> MarkdownBlock? {
+ guard looksLikeTableRow(headerLine),
+ let delimiter = remaining.first,
+ let alignments = delimiterAlignments(delimiter),
+ !alignments.isEmpty
+ else { return nil }
+
+ let headerCells = cells(headerLine)
+ guard headerCells.count == alignments.count else { return nil }
+
+ remaining = remaining.dropFirst()
+ let columns = alignments.count
+ let headers = padded(headerCells, to: columns, fill: "")
+ var rows: [[String]] = []
+ while let next = remaining.first {
+ let trimmed = next.trimmingCharacters(in: .whitespaces)
+ // A body row may look like a delimiter (`| --- | --- |`). GFM
+ // still treats it as data once the header delimiter is spent.
+ if trimmed.isEmpty || !looksLikeTableRow(trimmed) {
+ break
+ }
+ remaining = remaining.dropFirst()
+ rows.append(padded(cells(trimmed), to: columns, fill: ""))
+ }
+ return .table(headers: headers, alignments: alignments, rows: rows)
+ }
+
+ private static func looksLikeTableRow(_ trimmed: String) -> Bool {
+ trimmed.contains("|")
+ }
+
+ /// Split on unescaped `|`, dropping the empty ends a leading/trailing
+ /// pipe creates. `\|` stays inside the cell so a filename or regex
+ /// cannot shift later columns.
+ private static func cells(_ line: String) -> [String] {
+ let trimmed = line.trimmingCharacters(in: .whitespaces)
+ var parts: [String] = []
+ var current = ""
+ var escaped = false
+ for character in trimmed {
+ if escaped {
+ current.append(character)
+ escaped = false
+ continue
+ }
+ if character == "\\" {
+ escaped = true
+ continue
+ }
+ if character == "|" {
+ parts.append(current.trimmingCharacters(in: .whitespaces))
+ current = ""
+ continue
+ }
+ current.append(character)
+ }
+ if escaped { current.append("\\") }
+ parts.append(current.trimmingCharacters(in: .whitespaces))
+ if trimmed.hasPrefix("|"), parts.first == "" { parts.removeFirst() }
+ if trimmed.hasSuffix("|"), !trimmed.hasSuffix("\\|"), parts.last == "" { parts.removeLast() }
+ return parts
+ }
+
+ /// Every cell must be `---` / `:---` / `---:` / `:---:`. A lone `---`
+ /// with no pipe is a horizontal rule, not a delimiter — that path
+ /// never calls this because `looksLikeTableRow` requires a `|`.
+ private static func delimiterAlignments(_ line: String) -> [MarkdownTableAlignment]? {
+ let trimmed = line.trimmingCharacters(in: .whitespaces)
+ guard looksLikeTableRow(trimmed) else { return nil }
+ let parts = cells(trimmed)
+ guard !parts.isEmpty else { return nil }
+ var alignments: [MarkdownTableAlignment] = []
+ alignments.reserveCapacity(parts.count)
+ for part in parts {
+ guard let alignment = alignment(from: part) else { return nil }
+ alignments.append(alignment)
+ }
+ return alignments
+ }
+
+ private static func alignment(from cell: String) -> MarkdownTableAlignment? {
+ let left = cell.hasPrefix(":")
+ let right = cell.hasSuffix(":")
+ let dashes = cell.dropFirst(left ? 1 : 0).dropLast(right ? 1 : 0)
+ guard dashes.count >= 3, dashes.allSatisfy({ $0 == "-" }) else { return nil }
+ if left && right { return .center }
+ if right { return .right }
+ return .left
+ }
+
+ private static func padded(_ items: [T], to count: Int, fill: T) -> [T] {
+ if items.count >= count { return Array(items.prefix(count)) }
+ return items + Array(repeating: fill, count: count - items.count)
+ }
}
diff --git a/ios/Sources/CompanionCore/Models.swift b/ios/Sources/CompanionCore/Models.swift
index ede207ef08..ea3f56e442 100644
--- a/ios/Sources/CompanionCore/Models.swift
+++ b/ios/Sources/CompanionCore/Models.swift
@@ -265,6 +265,13 @@ public struct PairResponse: Codable, Sendable {
public var hosts: [String]?
}
+/// `POST /api/inbox` — the sidecar wrote this file onto the computer.
+public struct InboxFile: Codable, Sendable {
+ public var path: String
+ public var name: String
+ public var size: Int
+}
+
/// A freshly minted provider viewer. It is deliberately not Codable for
/// persistence: the URL is a short-lived bearer credential and belongs only
/// in memory for the browser session that requested it.
diff --git a/ios/TESTING.md b/ios/TESTING.md
index cba8a99bb7..f3809a0627 100644
--- a/ios/TESTING.md
+++ b/ios/TESTING.md
@@ -133,10 +133,10 @@ xcodebuild -project OpenMausCompanion.xcodeproj \
CODE_SIGNING_ALLOWED=NO build
```
-**Re-run `xcodegen generate` whenever a pull adds a file to `App/`.** The
-generated project lists source files explicitly, so a new one is missing from
-the target until you regenerate, and the build fails with `Cannot find 'X' in
-scope` — which looks like a code error and is not one.
+**Quit Xcode, then re-run `xcodegen generate` once after this pull** (`App/`
+is a synced folder; new Swift files after that do not need a regenerate).
+Pulling while Xcode is open is what produces "Build input files cannot be
+found" for files the old project listed.
### If the app is letterboxed inside black bars
@@ -203,7 +203,22 @@ On the phone, in order:
then come back. The transcript should catch up *without* a visible reload —
that is the resumable stream doing its job. Watch the harness log to confirm
it replayed rather than re-hydrated.
-6. **Revoke.** Remove the device in Settings → Companion on the computer. The
+6. **Dictate.** Open a chat, tap the mic, speak, tap the mic again. The words
+ should land in the field as you talk (not only after you stop), survive
+ an edit, and send like anything you typed. The first tap prompts for
+ Microphone and Speech Recognition; a denial names Settings → OpenMausMobile
+ on that same attempt. Backgrounding the app must stop the mic — lock the
+ phone mid-sentence and confirm it is not still listening when you come
+ back. Opening the computer panel must also release it (ChatView stays
+ mounted under the push).
+7. **Attach.** Open a chat, tap +, pick Attach Image / Take Photo / Choose
+ File. A chip should appear above the field. Send with or without a caption.
+ On the computer, the file lands under the sidecar inbox
+ (`~/.openmausbot-companion/inbox/`) and the bot's prompt contains
+ `` — the same tag a desktop drop produces. The
+ first Take Photo prompts for Camera. A simulator has no camera and should
+ say so rather than crash. Eight files, 8 MB each, is the ceiling.
+8. **Revoke.** Remove the device in Settings → Companion on the computer. The
phone should land on "This phone was unpaired" rather than silently failing.
---
@@ -257,7 +272,8 @@ Not built yet, so not bugs:
- **Nothing arrives after the app is terminated.** Live and replayed notification
frames now become native alerts and badges, but closed-app push still needs an
APNs relay with project-owned Apple credentials.
-- **No voice or routine management.** Tasks, SQLite transcript search/export,
+- **No call mode, spoken replies, or routine management.** Composer dictation
+ and photo/file attach are in. Tasks, SQLite transcript search/export,
reactions, and edit/version switching are available from the conversation UI.
(Two entries that used to sit on this list have since shipped: replies stream
diff --git a/ios/Tests/CompanionCoreTests/AttachmentTests.swift b/ios/Tests/CompanionCoreTests/AttachmentTests.swift
new file mode 100644
index 0000000000..7208019378
--- /dev/null
+++ b/ios/Tests/CompanionCoreTests/AttachmentTests.swift
@@ -0,0 +1,109 @@
+// Composer attachments: how typed text and host file paths share a prompt.
+//
+// The picker and the sidecar write live in App/ and companion/. The join
+// is the part with a decision in it — the same tagged path the desktop
+// composer already sends — and getting the escaping wrong is a path that
+// can break out of the attribute.
+import XCTest
+@testable import CompanionCore
+
+final class AttachmentTests: XCTestCase {
+ func testAPhotoWithNoCaptionIsStillAMessage() {
+ let file = Attachment.File(path: "/tmp/photo.jpg", name: "photo.jpg", size: 12)
+ XCTAssertEqual(
+ Attachment.draft(text: "", files: [file]),
+ ""
+ )
+ }
+
+ func testTypedTextComesFirst() {
+ let file = Attachment.File(path: "/tmp/a.txt", name: "a.txt", size: 1)
+ XCTAssertEqual(
+ Attachment.draft(text: "please look", files: [file]),
+ "please look\n\n"
+ )
+ }
+
+ func testEmptyEverythingStaysEmpty() {
+ XCTAssertEqual(Attachment.draft(text: " ", files: []), "")
+ }
+
+ func testSeveralFilesKeepTheirOrder() {
+ let files = [
+ Attachment.File(path: "/tmp/a.jpg", name: "a.jpg", size: 1),
+ Attachment.File(path: "/tmp/b.pdf", name: "b.pdf", size: 2),
+ ]
+ XCTAssertEqual(
+ Attachment.draft(text: "here", files: files),
+ "here\n\n\n\n"
+ )
+ }
+
+ func testAPathWithQuotesCannotLeaveTheAttribute() {
+ XCTAssertEqual(
+ Attachment.escapeAttribute("/tmp/a\"&<>\t\n.txt"),
+ "/tmp/a"&<>
.txt"
+ )
+ XCTAssertEqual(
+ Attachment.unescapeAttribute("/tmp/a"&<>
.txt"),
+ "/tmp/a\"&<>\t\n.txt"
+ )
+ }
+
+ func testDisplayHidesTheTagAndKeepsTheCaption() {
+ let shown = Attachment.display(
+ "Can create a listing\n\n"
+ )
+ XCTAssertEqual(shown.caption, "Can create a listing")
+ XCTAssertEqual(shown.files.map(\.name), ["1787025214436-54414f70-photo.jpg"])
+ XCTAssertEqual(
+ shown.files.first?.path,
+ "/Users/me/.openmausbot-companion/inbox/1787025214436-54414f70-photo.jpg"
+ )
+ XCTAssertTrue(shown.files.first?.isImage == true)
+ XCTAssertEqual(shown.files.first?.displayName, "photo.jpg")
+ }
+
+ func testAPhotoWithNoCaptionStillDisplaysAsAFile() {
+ let shown = Attachment.display("")
+ XCTAssertEqual(shown.caption, "")
+ XCTAssertEqual(shown.files.map(\.name), ["photo.jpg"])
+ }
+
+ func testAPlainMessageIsUnchanged() {
+ let shown = Attachment.display("just text")
+ XCTAssertEqual(shown.caption, "just text")
+ XCTAssertEqual(shown.files, [])
+ }
+
+ func testDisplayRoundTripsADraft() {
+ let files = [
+ Attachment.File(path: "/tmp/a.jpg", name: "a.jpg", size: 1),
+ Attachment.File(path: "/tmp/notes.txt", name: "notes.txt", size: 2),
+ ]
+ let shown = Attachment.display(Attachment.draft(text: "please look", files: files))
+ XCTAssertEqual(shown.caption, "please look")
+ XCTAssertEqual(shown.files.map(\.path), ["/tmp/a.jpg", "/tmp/notes.txt"])
+ XCTAssertEqual(shown.files.map(\.name), ["a.jpg", "notes.txt"])
+ XCTAssertEqual(shown.files.map(\.isImage), [true, false])
+ }
+
+ func testAPdfIsNotAnImage() {
+ XCTAssertFalse(Attachment.File(path: "/tmp/a.pdf", name: "a.pdf", size: 1).isImage)
+ }
+
+ func testInboxDisplayNameDropsTheUniquenessPrefix() {
+ XCTAssertEqual(
+ Attachment.File(
+ path: "/tmp/1787025214436-54414f70-photo.jpg",
+ name: "1787025214436-54414f70-photo.jpg",
+ size: 1
+ ).displayName,
+ "photo.jpg"
+ )
+ XCTAssertEqual(
+ Attachment.File(path: "/tmp/notes.txt", name: "notes.txt", size: 1).displayName,
+ "notes.txt"
+ )
+ }
+}
diff --git a/ios/Tests/CompanionCoreTests/DictationTests.swift b/ios/Tests/CompanionCoreTests/DictationTests.swift
new file mode 100644
index 0000000000..5140f29649
--- /dev/null
+++ b/ios/Tests/CompanionCoreTests/DictationTests.swift
@@ -0,0 +1,82 @@
+// Composer dictation: how typed text and a live transcript share a field.
+//
+// The Speech session is in App/ and needs a device. The join is the part
+// with a decision in it — partials replace, they do not stack — and getting
+// that wrong is a composer that writes "hello hello hello world" as you
+// talk. Same shape as the desktop: freeze the base when the mic goes on,
+// and every subsequent transcript is `base + spoken`.
+import XCTest
+@testable import CompanionCore
+
+final class DictationTests: XCTestCase {
+ func testEmptyComposerTakesTheTranscript() {
+ XCTAssertEqual(Dictation.draft(base: "", transcript: "hello"), "hello")
+ }
+
+ func testEmptyTranscriptLeavesTheBase() {
+ // The first callback has not arrived yet. Wiping the field in that
+ // window would look like the mic deleted what you had typed.
+ XCTAssertEqual(Dictation.draft(base: "please look", transcript: ""), "please look")
+ XCTAssertEqual(Dictation.draft(base: "please look", transcript: " "), "please look")
+ }
+
+ func testSpokenTextAppendsAfterTypedText() {
+ XCTAssertEqual(Dictation.draft(base: "please", transcript: "look at the logs"), "please look at the logs")
+ }
+
+ func testWhitespaceAroundEitherSideIsTrimmed() {
+ XCTAssertEqual(Dictation.draft(base: " please ", transcript: " look "), "please look")
+ }
+
+ /// The contract ChatView has to keep: the base is the text at the
+ /// moment listening started, not the live draft. Re-joining against
+ /// that frozen base is how a later partial replaces an earlier one
+ /// instead of concatenating onto it.
+ func testALaterPartialReplacesAnEarlierOne() {
+ let base = "please"
+ XCTAssertEqual(Dictation.draft(base: base, transcript: "look"), "please look")
+ XCTAssertEqual(Dictation.draft(base: base, transcript: "look at the logs"), "please look at the logs")
+ }
+
+ func testBothEmptyStaysEmpty() {
+ XCTAssertEqual(Dictation.draft(base: "", transcript: ""), "")
+ XCTAssertEqual(Dictation.draft(base: " ", transcript: "\n"), "")
+ }
+
+ // MARK: - Locale candidates
+
+ func testPreferredLanguageComesFirst() {
+ let locales = Dictation.localeCandidates(
+ preferredLanguages: ["fr-FR", "de-DE"],
+ current: Locale(identifier: "en-US")
+ )
+ XCTAssertEqual(Dictation.canonicalIdentifier(locales[0]), "fr_fr")
+ XCTAssertTrue(locales.map(Dictation.canonicalIdentifier).contains("en_us"))
+ }
+
+ func testEnglishIsNotDuplicatedWhenItIsAlreadyPreferred() {
+ let locales = Dictation.localeCandidates(
+ preferredLanguages: ["en-US"],
+ current: Locale(identifier: "en-US")
+ )
+ let keys = locales.map(Dictation.canonicalIdentifier)
+ XCTAssertEqual(keys, ["en_us"])
+ }
+
+ func testHyphenAndUnderscoreAreTheSameCandidate() {
+ let locales = Dictation.localeCandidates(
+ preferredLanguages: ["en-US"],
+ current: Locale(identifier: "en_US")
+ )
+ XCTAssertEqual(locales.map(Dictation.canonicalIdentifier), ["en_us"])
+ }
+
+ func testEnglishIsTheLastResortWhenNothingElseIsOffered() {
+ let locales = Dictation.localeCandidates(
+ preferredLanguages: [],
+ current: Locale(identifier: "ja-JP")
+ )
+ XCTAssertEqual(Dictation.canonicalIdentifier(locales[0]), "ja_jp")
+ XCTAssertEqual(Dictation.canonicalIdentifier(try XCTUnwrap(locales.last)), "en_us")
+ }
+}
diff --git a/ios/Tests/CompanionCoreTests/MarkdownTests.swift b/ios/Tests/CompanionCoreTests/MarkdownTests.swift
index fc342cef34..2f1c354c60 100644
--- a/ios/Tests/CompanionCoreTests/MarkdownTests.swift
+++ b/ios/Tests/CompanionCoreTests/MarkdownTests.swift
@@ -154,12 +154,177 @@ final class MarkdownTests: XCTestCase {
XCTAssertEqual(Markdown.blocks("-- dashes --"), [.paragraph("-- dashes --")])
}
+ // MARK: - Tables
+
+ func testPipeTableKeepsHeadersRowsAndAlignment() {
+ let source = """
+ | # | Circuit | # | Circuit |
+ |---|:-------:|---:|:--------|
+ | 1 | Oven | 2 | Dryer |
+ | 3 | A/C | 4 | Kitchen Rec |
+ """
+ XCTAssertEqual(
+ Markdown.blocks(source),
+ [
+ .table(
+ headers: ["#", "Circuit", "#", "Circuit"],
+ alignments: [.left, .center, .right, .left],
+ rows: [
+ ["1", "Oven", "2", "Dryer"],
+ ["3", "A/C", "4", "Kitchen Rec"],
+ ]
+ ),
+ ]
+ )
+ }
+
+ /// The panel-directory shape models actually emit: two `#` / Circuit
+ /// pairs with an empty spacer cell between them.
+ func testABlankCellBetweenColumnPairsIsKept() {
+ let source = """
+ | # | Circuit | | # | Circuit |
+ |---|----------|---|---|----------|
+ | 1 | Oven | | 2 | Dryer |
+ """
+ XCTAssertEqual(
+ Markdown.blocks(source),
+ [
+ .table(
+ headers: ["#", "Circuit", "", "#", "Circuit"],
+ alignments: [.left, .left, .left, .left, .left],
+ rows: [["1", "Oven", "", "2", "Dryer"]],
+ ),
+ ]
+ )
+ }
+
+ /// Without a delimiter row this is prose that happens to contain pipes,
+ /// which is how it rendered as a wall of `|` before tables existed here.
+ func testPipesWithoutADelimiterStayAParagraph() {
+ XCTAssertEqual(
+ Markdown.blocks("| # | Circuit |\n| 1 | Oven |"),
+ [.paragraph("| # | Circuit | | 1 | Oven |")]
+ )
+ }
+
+ func testAHeaderWithoutADelimiterIsStillProse() {
+ XCTAssertEqual(Markdown.blocks("| # | Circuit |"), [.paragraph("| # | Circuit |")])
+ }
+
+ /// GFM: header and delimiter must have the same number of cells.
+ /// A mismatch is not a table — padding the header down would drop
+ /// characters from a reply that is still streaming its delimiter.
+ func testMismatchedHeaderAndDelimiterStayProse() {
+ let source = """
+ | a | b | c |
+ | --- | --- |
+ | 1 | 2 | 3 |
+ | only |
+ """
+ let rendered = Markdown.blocks(source)
+ XCTAssertEqual(rendered.count, 1)
+ if case let .paragraph(text) = rendered[0] {
+ XCTAssertTrue(text.contains("a"))
+ XCTAssertTrue(text.contains("c"))
+ XCTAssertTrue(text.contains("only"))
+ } else {
+ XCTFail("expected a paragraph, got \(rendered)")
+ }
+ }
+
+ func testADashOnlyBodyRowStaysInTheTable() {
+ let source = """
+ | a | b |
+ | --- | --- |
+ | --- | --- |
+ | 1 | 2 |
+ """
+ XCTAssertEqual(
+ Markdown.blocks(source),
+ [
+ .table(
+ headers: ["a", "b"],
+ alignments: [.left, .left],
+ rows: [
+ ["---", "---"],
+ ["1", "2"],
+ ]
+ ),
+ ]
+ )
+ }
+
+ func testEscapedPipesStayInsideTheCell() {
+ let source = """
+ | A\\|B | C |
+ | --- | --- |
+ | 1\\|2 | 3 |
+ """
+ XCTAssertEqual(
+ Markdown.blocks(source),
+ [
+ .table(
+ headers: ["A|B", "C"],
+ alignments: [.left, .left],
+ rows: [["1|2", "3"]]
+ ),
+ ]
+ )
+ }
+
+ func testATableStopsAtABlankLine() {
+ let source = """
+ | a | b |
+ | --- | --- |
+ | 1 | 2 |
+
+ after
+ """
+ XCTAssertEqual(
+ Markdown.blocks(source),
+ [
+ .table(
+ headers: ["a", "b"],
+ alignments: [.left, .left],
+ rows: [["1", "2"]]
+ ),
+ .paragraph("after"),
+ ]
+ )
+ }
+
+ func testAPartialTableStillHasItsHeader() {
+ XCTAssertEqual(
+ Markdown.blocks("| a | b |\n| --- | --- |"),
+ [
+ .table(
+ headers: ["a", "b"],
+ alignments: [.left, .left],
+ rows: []
+ ),
+ ]
+ )
+ }
+
+ func testInlineMarkupInCellsSurvivesTheSplit() {
+ XCTAssertEqual(
+ Markdown.blocks("| **Name** |\n| --- |\n| `Oven` |"),
+ [
+ .table(
+ headers: ["**Name**"],
+ alignments: [.left],
+ rows: [["`Oven`"]]
+ ),
+ ]
+ )
+ }
+
// MARK: - Streaming
/// The invariant that keeps the bubble from flickering: whatever arrives,
/// something renders, and the characters the model has sent are in it.
func testPartialInputAlwaysRendersSomething() {
- for prefix in ["#", "# ", "# Head", "- ", "- it", "**bo", "```", "```sw\nlet", "[link](htt"] {
+ for prefix in ["#", "# ", "# Head", "- ", "- it", "**bo", "```", "```sw\nlet", "[link](htt", "| a |"] {
XCTAssertFalse(
Markdown.blocks(prefix).isEmpty,
"dropped everything for \(prefix.debugDescription)"
@@ -170,15 +335,16 @@ final class MarkdownTests: XCTestCase {
/// Growing the source one character at a time must never lose text. This
/// is the whole stream, replayed at the granularity the deltas arrive at.
func testNoPrefixOfAReplyLosesCharacters() {
- let reply = "# Result\n\nRan **two** checks:\n\n- `pnpm test` passed\n- `pnpm lint` passed\n\n```sh\npnpm test\n```\n\n> nothing else to report"
+ let reply = "# Result\n\nRan **two** checks:\n\n- `pnpm test` passed\n- `pnpm lint` passed\n\n| check | ok |\n| --- | --- |\n| test | yes |\n\n```sh\npnpm test\n```\n\n> nothing else to report"
for length in 1...reply.count {
let partial = String(reply.prefix(length))
let rendered = Markdown.blocks(partial).map(text).joined()
// Compare on non-whitespace: the splitter deliberately drops
// markers, indentation and blank lines, and it joins soft breaks
// with a space. What it must not drop is content.
- let sent = partial.filter { !$0.isWhitespace && !"#->`*_".contains($0) }
- let shown = rendered.filter { !$0.isWhitespace && !"#->`*_".contains($0) }
+ let dropped: Set = ["#", "-", ">", "`", "*", "_", "|"]
+ let sent = partial.filter { !$0.isWhitespace && !dropped.contains($0) }
+ let shown = rendered.filter { !$0.isWhitespace && !dropped.contains($0) }
XCTAssertEqual(shown, sent, "lost content at \(length) characters")
}
}
@@ -191,6 +357,7 @@ final class MarkdownTests: XCTestCase {
case let .heading(_, text): return text
case let .code(language, text): return (language ?? "") + text
case let .quote(text): return text
+ case let .table(headers, _, rows): return headers.joined() + rows.flatMap { $0 }.joined()
case .rule: return ""
}
}
diff --git a/ios/project.yml b/ios/project.yml
index 129bf89f1a..cbaf62d162 100644
--- a/ios/project.yml
+++ b/ios/project.yml
@@ -13,6 +13,12 @@ options:
deploymentTarget:
iOS: "17.0"
createIntermediateGroups: true
+ # Xcode 16 buildable folder: compile whatever is in App/ on disk.
+ # The old `group` type listed every file in the pbxproj, so a pull that
+ # landed while Xcode was open (or an iCloud Documents eviction) produced
+ # "Build input files cannot be found" for files the project remembered
+ # and the working tree did not. Needs XcodeGen 2.44+.
+ projectFormat: xcode16_0
packages:
CompanionCore:
@@ -24,6 +30,7 @@ targets:
platform: iOS
sources:
- path: App
+ type: syncedFolder
dependencies:
- package: CompanionCore
product: CompanionCore
@@ -95,3 +102,17 @@ targets:
ts.net:
NSIncludesSubdomains: true
NSExceptionAllowsInsecureHTTPLoads: true
+ # Composer dictation. Without these two the first tap on the mic
+ # crashes rather than prompting, which looks like the app is broken
+ # rather than missing a string. Same keys the desktop speech-helper
+ # carries, worded for the phone.
+ NSMicrophoneUsageDescription: >-
+ OpenMausMobile listens while you dictate a message to a bot.
+ NSSpeechRecognitionUsageDescription: >-
+ OpenMausMobile converts your speech into text so you can send it
+ to a bot. Recognition runs on this phone when it can.
+ # Take Photo in the composer. PhotosPicker (the library) does not
+ # need a library string; the camera still does, and without it the
+ # first tap crashes rather than prompting.
+ NSCameraUsageDescription: >-
+ OpenMausMobile uses the camera so you can send a photo to a bot.