Integrate #58: safe structured computer observations

package.json

@@ -27,6 +27,7 @@
     "build": "tsc -b && tsc -p tsconfig.server.json && vite build",
     "typecheck": "tsc -b && tsc -p tsconfig.server.json",
     "test": "vitest run",
+    "bench:observation": "node --experimental-strip-types scripts/bench-observation.ts",
     "test:watch": "vitest",
     "check:electron": "node --check electron/main.mjs && node --check electron/preload.cjs && node --check electron/capabilities.cjs && node --check electron/cua-connection.cjs && node --check electron/cua.mjs && node --check electron/speech.mjs",
     "preview": "vite preview",

scripts/bench-observation.ts

@@ -0,0 +1,23 @@
+import { ObservationCoordinator } from "../server/computer-observation.ts";
+
+// Stable fixture: every requested observation still captures, because pages
+// can update asynchronously. Only byte-identical frames are withheld from the
+// model; deterministic actions therefore always receive fresh evidence.
+const frames = ["initial", "initial", "after-click", "after-click", "after-submit", "after-submit", "after-submit"];
+const baselineSent = frames.length;
+const coordinator = new ObservationCoordinator();
+for (const [index, frame] of frames.entries()) {
+  // index 1 deliberately models an action whose pixels do not change: it
+  // must still capture once, while duplicate suppression avoids a model send.
+  if (index === 1 || index === 2 || index === 4) coordinator.noteAction();
+  coordinator.observeFrame(frame, null);
+}
+const metrics = coordinator.metrics;
+const reduction = ((1 - metrics.screenshotsSentToModel / baselineSent) * 100).toFixed(1);
+const suppressed = metrics.screenshotsCaptured - metrics.screenshotsSentToModel;
+if (suppressed < 1 || metrics.screenshotsCaptured !== frames.length) {
+  throw new Error("fixture failed to capture every observation or suppress a duplicate");
+}
+console.log("Computer observation benchmark (offline deterministic fixture)");
+console.log(`Screenshot observations sent to model: ${baselineSent} → ${metrics.screenshotsSentToModel} (${reduction}% reduction)`);
+console.log(`Actions: ${metrics.computerActions}; screenshots captured: ${metrics.screenshotsCaptured}; duplicate captures suppressed: ${suppressed}`);

server/computer-observation.test.ts

@@ -0,0 +1,67 @@
+import { describe, expect, it } from "vitest";
+import {
+  normalizeBrowserUrl,
+  normalizeCrop,
+  ObservationCoordinator,
+  parseBrowserTargets,
+  safeBrowserUrl,
+} from "./computer-observation.ts";
+
+describe("computer observation coordinator", () => {
+  it("captures after actions but sends vision only for changed pixels", () => {
+    const coordinator = new ObservationCoordinator();
+    expect(coordinator.observeFrame("frame-a", null)).toMatchObject({ changed: true });
+    coordinator.noteAction();
+    expect(coordinator.observeFrame("frame-a", null)).toMatchObject({ changed: false });
+    coordinator.noteAction();
+    expect(coordinator.observeFrame("frame-b", { x: 20, y: 20, width: 100, height: 100 })).toMatchObject({ changed: true });
+    coordinator.noteRetry();
+    coordinator.noteStructuredObservation();
+    coordinator.noteVerification(true);
+    coordinator.noteVerification(false);
+    expect(coordinator.metrics).toEqual({
+      screenshotsCaptured: 3,
+      screenshotsSentToModel: 2,
+      fullScreenObservations: 1,
+      croppedObservations: 1,
+      structuredBrowserObservations: 1,
+      computerActions: 2,
+      retries: 1,
+      verificationSuccesses: 1,
+      verificationFailures: 1,
+    });
+  });
+
+  it("accepts bounded crop regions and rejects width or height overflow", () => {
+    expect(normalizeCrop({ x: 10, y: 20, width: 200, height: 100 }, 1280, 720)).toEqual({
+      x: 10,
+      y: 20,
+      width: 200,
+      height: 100,
+    });
+    expect(normalizeCrop({ x: -1, y: 0, width: 200, height: 100 }, 1280, 720)).toBeNull();
+    expect(normalizeCrop({ x: 0, y: 0, width: 31, height: 100 }, 1280, 720)).toBeNull();
+    expect(normalizeCrop({ x: 0, y: 0, width: 100, height: 31 }, 1280, 720)).toBeNull();
+    expect(normalizeCrop({ x: 0, y: 0, width: "wide", height: 100 }, 1280, 720)).toBeNull();
+    expect(normalizeCrop({ x: 1200, y: 0, width: 200, height: 100 }, 1280, 720)).toBeNull();
+    expect(normalizeCrop({ x: 0, y: 700, width: 100, height: 50 }, 1280, 720)).toBeNull();
+  });
+
+  it("redacts exposed URLs but preserves full navigation state internally", () => {
+    const raw = "https://user:password@example.com/a?token=secret#fragment";
+    expect(safeBrowserUrl(raw)).toBe("https://example.com/a");
+    expect(normalizeBrowserUrl(raw)).toBe("https://example.com/a?token=secret#fragment");
+    expect(safeBrowserUrl(`https://example.com/${"a".repeat(2_048)}`)).toBeNull();
+    expect(parseBrowserTargets(JSON.stringify([
+      { id: "one", type: "page", title: " Example page ", url: raw },
+      { id: "two", type: "service_worker", url: "https://example.com/worker" },
+    ]))).toEqual([
+      {
+        id: "one",
+        title: "Example page",
+        url: "https://example.com/a",
+        comparisonUrl: "https://example.com/a?token=secret#fragment",
+      },
+    ]);
+  });
+});

server/computer-observation.ts

@@ -0,0 +1,159 @@
+import { createHash } from "node:crypto";
+
+/** Provider-neutral policy for deciding when a computer observation needs vision. */
+export interface ObservationMetrics {
+  screenshotsCaptured: number;
+  screenshotsSentToModel: number;
+  fullScreenObservations: number;
+  croppedObservations: number;
+  structuredBrowserObservations: number;
+  computerActions: number;
+  retries: number;
+  verificationSuccesses: number;
+  verificationFailures: number;
+}
+
+export interface CropRegion {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export interface BrowserTarget {
+  id: string;
+  title: string;
+  /** Safe for a model or log: credentials, query, and fragment removed. */
+  url: string;
+  /** Internal-only comparison value. Never include this in tool output. */
+  comparisonUrl: string;
+}
+
+export const emptyObservationMetrics = (): ObservationMetrics => ({
+  screenshotsCaptured: 0,
+  screenshotsSentToModel: 0,
+  fullScreenObservations: 0,
+  croppedObservations: 0,
+  structuredBrowserObservations: 0,
+  computerActions: 0,
+  retries: 0,
+  verificationSuccesses: 0,
+  verificationFailures: 0,
+});
+
+export function normalizeCrop(raw: unknown, maxWidth: number, maxHeight: number): CropRegion | null {
+  if (!raw || typeof raw !== "object") return null;
+  const value = raw as Record<string, unknown>;
+  const x = Math.round(Number(value.x));
+  const y = Math.round(Number(value.y));
+  const width = Math.round(Number(value.width));
+  const height = Math.round(Number(value.height));
+  if (
+    ![x, y, width, height, maxWidth, maxHeight].every(Number.isFinite) ||
+    maxWidth <= 0 ||
+    maxHeight <= 0 ||
+    x < 0 ||
+    y < 0 ||
+    width < 32 ||
+    height < 32
+  ) {
+    return null;
+  }
+  if (x + width > maxWidth || y + height > maxHeight) return null;
+  return { x, y, width, height };
+}
+
+/** Canonical value for internal navigation checks. Credentials are never
+ * needed for equality and are removed here; query and fragment remain so
+ * two distinct application states cannot verify as the same destination. */
+export function normalizeBrowserUrl(value: unknown): string | null {
+  if (typeof value !== "string" || !value || value.length > 8_192) return null;
+  try {
+    const url = new URL(value);
+    if (!/^https?:$/.test(url.protocol)) return null;
+    url.username = "";
+    url.password = "";
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
+/** Removes credentials, query, and fragment before browser state reaches a model or log. */
+export function safeBrowserUrl(value: unknown): string | null {
+  const normalized = normalizeBrowserUrl(value);
+  if (!normalized) return null;
+  const url = new URL(normalized);
+  url.search = "";
+  url.hash = "";
+  const safe = url.toString();
+  return safe.length <= 2_048 ? safe : null;
+}
+
+/** Parses Chrome's /json/list response into a small, safe structured observation. */
+export function parseBrowserTargets(raw: string): BrowserTarget[] {
+  if (raw.length > 1_000_000) return [];
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    if (!Array.isArray(parsed)) return [];
+    return parsed.slice(0, 20).flatMap((item) => {
+      if (!item || typeof item !== "object") return [];
+      const value = item as Record<string, unknown>;
+      const comparisonUrl = normalizeBrowserUrl(value.url);
+      const url = safeBrowserUrl(value.url);
+      if (value.type !== "page" || !url || !comparisonUrl || typeof value.id !== "string") return [];
+      const title = typeof value.title === "string" ? value.title.replace(/\s+/g, " ").trim().slice(0, 200) : "";
+      return [{ id: value.id.slice(0, 100), title, url, comparisonUrl }];
+    });
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Keeps observations cheap without claiming the screen is immutable. Every
+ * requested observation still captures fresh pixels (pages can change without
+ * an input action), while byte-identical frames are not sent to the model twice.
+ */
+export class ObservationCoordinator {
+  metrics = emptyObservationMetrics();
+  private lastObservation: string | null = null;
+
+  noteAction(count = 1) {
+    this.metrics.computerActions += Math.max(0, Math.trunc(count));
+  }
+
+  noteRetry() {
+    this.metrics.retries += 1;
+  }
+
+  /** canonicalFrame must describe the full screenshot, even when the image
+   * returned to the model is cropped. A box-provided full-frame hash works. */
+  observeFrame(canonicalFrame: string | null, crop: CropRegion | null) {
+    this.metrics.screenshotsCaptured += 1;
+    const hash = canonicalFrame
+      ? createHash("sha256").update(canonicalFrame).digest("hex")
+      : null;
+    const view = crop ? `${crop.x},${crop.y},${crop.width},${crop.height}` : "full";
+    const signature = hash ? `${hash}:${view}` : null;
+    // If the box cannot provide a full-frame hash, fail open and send the
+    // valid image. Suppressing a possibly-new crop would be worse.
+    const changed = signature === null || signature !== this.lastObservation;
+    if (signature) this.lastObservation = signature;
+    if (changed) {
+      this.metrics.screenshotsSentToModel += 1;
+      if (crop) this.metrics.croppedObservations += 1;
+      else this.metrics.fullScreenObservations += 1;
+    }
+    return { changed, hash };
+  }
+
+  noteStructuredObservation() {
+    this.metrics.structuredBrowserObservations += 1;
+  }
+
+  noteVerification(ok: boolean) {
+    if (ok) this.metrics.verificationSuccesses += 1;
+    else this.metrics.verificationFailures += 1;
+  }
+}