test(cu): measure a rendering change against recorded trajectories, and name what a model got stuck on

package.json

@@ -31,7 +31,7 @@
     "test:dist": "npm run test:scripts:full && node scripts/run-workspace-tests-parallel.mjs --concurrency=3",
     "test:dist:serial": "npm run test:scripts:full && node scripts/run-workspace-tests-parallel.mjs --serial",
     "test:fast": "npm run build:test && npm run test:scripts && node scripts/run-workspace-tests-parallel.mjs --concurrency=3",
-    "test:scripts": "node --test scripts/fixture-env.test.mjs scripts/electron-lifecycle.test.mjs scripts/check-story-annotations.test.mjs scripts/ci-test-plan.test.mjs scripts/run-headless-tests.test.mjs scripts/run-workspace-tests-parallel.test.mjs scripts/cu-e2e-scenarios.test.mjs scripts/cu-report-sanitize.test.mjs scripts/computer-use-provenance.test.mjs apps/desktop/scripts/dev-app-runtime.test.mjs",
+    "test:scripts": "node --test scripts/fixture-env.test.mjs scripts/electron-lifecycle.test.mjs scripts/check-story-annotations.test.mjs scripts/ci-test-plan.test.mjs scripts/run-headless-tests.test.mjs scripts/run-workspace-tests-parallel.test.mjs scripts/cu-e2e-scenarios.test.mjs scripts/cu-report-sanitize.test.mjs scripts/computer-use-provenance.test.mjs scripts/cu-trace-analyse.test.mjs apps/desktop/scripts/dev-app-runtime.test.mjs",
     "test:scripts:extended": "node --test scripts/cu-provider-matrix.test.mjs scripts/cu-process-restart-harness.test.mjs scripts/cu-real-model-launcher.test.mjs scripts/macos-arm64-release.test.mjs scripts/measure-session-bundle.test.mjs",
     "test:scripts:full": "npm run test:scripts && npm run test:scripts:extended",
     "dev": "npm --workspace @maka/desktop run dev:hmr --",

packages/core/src/computer-use.ts

@@ -155,6 +155,96 @@ export const CU_ACTION_TYPES = [
 export const COMPUTER_USE_ACTION_TYPES = CU_ACTION_TYPES;
 export type CuActionType = (typeof CU_ACTION_TYPES)[number];
 
+/**
+ * The semantic actions, which name an element rather than a pixel. They are not
+ * in `CU_ACTION_TYPES` because they are not `CuAction`s — they are dispatched
+ * through `runSemantic` — but they are on the same wire enum, so anything
+ * reasoning about "what the model may ask for" has to see both halves.
+ */
+export const CU_SEMANTIC_ACTION_TYPES = [
+  'click_element',
+  'set_value',
+  'select_text',
+  'secondary_action',
+  'press_key',
+] as const;
+export type CuSemanticActionType = (typeof CU_SEMANTIC_ACTION_TYPES)[number];
+
+/**
+ * Every action name the tool schema spells out itself, as it spells them —
+ * that is, every name that is not one of the `CU_ACTION_TYPES` coordinate
+ * actions folded into `CU_TOOL_ACTION_TYPES` below.
+ *
+ * This list used to be hand-written beside a schema that already listed the
+ * same names, and it drifted: `window_action` was added to the strict union and
+ * not here, so had it also reached the wire, every window move, resize and
+ * minimise would have been summarised as `unknown` — in the approval a person
+ * reads before allowing it, and in the record the model reads back of its own
+ * call.
+ *
+ * Drift in the other direction costs just as much and is quieter. A name listed
+ * here that the schema does not accept makes `computerUseApprovalSummary`
+ * report an action the tool will reject as though it were one that had been
+ * taken, and makes `rememberForTurnAllowed` true for it. So the guard in
+ * `computer-use-schema-parity.test.ts` (@maka/runtime, which can import the
+ * schema; this package cannot) compares the two lists in both directions.
+ *
+ * It is no longer hand-written: the two openers are named here and the rest is
+ * spliced from `CU_SEMANTIC_ACTION_TYPES`, so there is one place to add a
+ * semantic action rather than two that must agree.
+ */
+export const COMPUTER_USE_SEMANTIC_ACTIONS = [
+  'list_apps',
+  'observe',
+  ...CU_SEMANTIC_ACTION_TYPES,
+] as const;
+
+/**
+ * Every action name the `maka_computer` tool accepts, in wire order.
+ *
+ * One list, so that adding an action cannot leave a consumer silently matching
+ * nothing. This has already cost us once: an offline analyser restated the
+ * vocabulary as two regexes, neither of which matched a single coordinate
+ * action after the surface moved, and it reported clean runs for trajectories
+ * made entirely of blind clicks.
+ */
+export const CU_TOOL_ACTION_TYPES = [...COMPUTER_USE_SEMANTIC_ACTIONS, ...CU_ACTION_TYPES] as const;
+export type CuToolActionType = (typeof CU_TOOL_ACTION_TYPES)[number];
+
+/**
+ * The actions that read without changing anything, and so take an observation
+ * lease rather than an action lease. Everything else on the wire is treated as
+ * mutating — including any action added later, which fails loud in an analyser
+ * rather than silently dropping out of the counts.
+ */
+export const CU_OBSERVING_ACTION_TYPES = [
+  'list_apps',
+  'observe',
+  'screenshot',
+  'cursor_position',
+  'wait',
+] as const;
+export type CuObservingActionType = (typeof CU_OBSERVING_ACTION_TYPES)[number];
+
+const OBSERVING_ACTION_SET: ReadonlySet<string> = new Set(CU_OBSERVING_ACTION_TYPES);
+const TOOL_ACTION_SET: ReadonlySet<string> = new Set(CU_TOOL_ACTION_TYPES);
+
+export const CU_MUTATING_ACTION_TYPES: readonly CuToolActionType[] = CU_TOOL_ACTION_TYPES.filter(
+  (action) => !OBSERVING_ACTION_SET.has(action),
+);
+
+export function isCuToolAction(action: string): action is CuToolActionType {
+  return TOOL_ACTION_SET.has(action);
+}
+
+export function isCuObservingAction(action: string): action is CuObservingActionType {
+  return OBSERVING_ACTION_SET.has(action);
+}
+
+export function isCuMutatingAction(action: string): action is CuToolActionType {
+  return TOOL_ACTION_SET.has(action) && !OBSERVING_ACTION_SET.has(action);
+}
+
 export type CuAction =
   | { type: 'screenshot' }
   | { type: 'cursor_position' }
@@ -466,35 +556,9 @@ const POINTER_ACTIONS = new Set([
 const KEYBOARD_ACTIONS = new Set(['type', 'key', 'hold_key', 'press_key']);
 const SEMANTIC_ACTIONS = new Set(['click_element', 'set_value', 'select_text', 'secondary_action']);
 
-/**
- * Every action name the tool schema spells out itself, as it spells them —
- * that is, every name that is not one of the `CU_ACTION_TYPES` coordinate
- * actions folded in below.
- *
- * Hand-written beside a schema that already lists them, and it drifted:
- * `window_action` was added to the strict union and not here, so had it also
- * reached the wire, every window move, resize and minimise would have been
- * summarised as `unknown` — in the approval a person reads before allowing it,
- * and in the record the model reads back of its own call.
- *
- * Drift in the other direction costs just as much and is quieter. A name listed
- * here that the schema does not accept makes `computerUseApprovalSummary`
- * report an action the tool will reject as though it were one that had been
- * taken, and makes `rememberForTurnAllowed` true for it. So the guard in
- * `computer-use-schema-parity.test.ts` (@maka/runtime, which can import the
- * schema; this package cannot) compares the two lists in both directions.
- */
-export const COMPUTER_USE_SEMANTIC_ACTIONS = [
-  'list_apps',
-  'observe',
-  'click_element',
-  'set_value',
-  'select_text',
-  'secondary_action',
-  'press_key',
-] as const;
-
-const APPROVAL_ACTIONS = new Set<string>([...COMPUTER_USE_SEMANTIC_ACTIONS, ...CU_ACTION_TYPES]);
+// Exactly the wire vocabulary, derived rather than restated: an action the tool
+// accepts is an action a person can be asked to approve.
+const APPROVAL_ACTIONS = new Set<string>(CU_TOOL_ACTION_TYPES);
 
 export function computerUseApprovalSummary(args: unknown): ComputerUseApprovalSummary {
   const record = asRecord(args);

packages/core/src/index.ts

@@ -599,9 +599,12 @@ export type {
   ComputerUseWindowIdentity,
   CuAction,
   CuActionType,
+  CuObservingActionType,
   CuPoint,
   CuRegion,
   CuScrollDirection,
+  CuSemanticActionType,
+  CuToolActionType,
 } from './computer-use.js';
 export {
   COMPUTER_USE_ACTION_TYPES,
@@ -612,11 +615,18 @@ export {
   COMPUTER_USE_ERROR_CODES,
   COMPUTER_USE_FRAME_SOURCE_KINDS,
   CU_ACTION_TYPES,
+  CU_MUTATING_ACTION_TYPES,
+  CU_OBSERVING_ACTION_TYPES,
   CU_SCROLL_DIRECTIONS,
+  CU_SEMANTIC_ACTION_TYPES,
+  CU_TOOL_ACTION_TYPES,
   computerUseApprovalScopeKey,
   computerUseApprovalSummary,
   computerUseModelCallArgs,
   isComputerUseErrorCode,
+  isCuMutatingAction,
+  isCuObservingAction,
+  isCuToolAction,
 } from './computer-use.js';
 
 // permission-profile.ts

packages/runtime/src/computer-use-tools.ts

@@ -8,8 +8,10 @@
 // threading (S18). The backend owns the actual AX/capture dispatch.
 import { z } from 'zod';
 import {
-  CU_ACTION_TYPES,
+  CU_TOOL_ACTION_TYPES,
   isComputerUseErrorCode,
+  isCuMutatingAction,
+  isCuObservingAction,
   type CuAction,
   type CuPoint,
   type ComputerUseErrorCode,
@@ -88,16 +90,7 @@ export type {
 export const computerWireParams = z
   .object({
     action: z
-      .enum([
-        'list_apps',
-        'observe',
-        'click_element',
-        'set_value',
-        'select_text',
-        'secondary_action',
-        'press_key',
-        ...CU_ACTION_TYPES,
-      ] as [string, ...string[]])
+      .enum(CU_TOOL_ACTION_TYPES as unknown as [string, ...string[]])
       .describe(
         'Operation to perform. Required fields by action: observe/screenshot require app or window_id; click_element requires observation_id and element_id; set_value requires observation_id, element_id, and value; select_text/secondary_action require observation_id, element_id, and text; press_key requires observation_id and text; coordinate actions require observation_id plus their coordinate fields.',
       ),
@@ -865,36 +858,15 @@ export function buildComputerUseTools(deps: {
             state.screenLocked();
             return sessionFailure('screen_locked');
           }
-          const requiresObservationLease =
-            input.action === 'observe' ||
-            input.action === 'screenshot' ||
-            input.action === 'list_apps' ||
-            input.action === 'cursor_position' ||
-            input.action === 'wait';
+          // Both halves of the wire enum are partitioned in `@maka/core`, so a
+          // new action cannot be added without landing on one side or the
+          // other — and offline consumers read the same partition.
+          const requiresObservationLease = isCuObservingAction(input.action);
           const observationLease = requiresObservationLease ? state.beforeObservation() : undefined;
           if (observationLease && !observationLease.ok) {
             return sessionFailure(observationLease.reason);
           }
-          const requiresActionLease =
-            input.action === 'click_element' ||
-            input.action === 'set_value' ||
-            input.action === 'select_text' ||
-            input.action === 'secondary_action' ||
-            input.action === 'press_key' ||
-            input.action === 'mouse_move' ||
-            input.action === 'left_click' ||
-            input.action === 'right_click' ||
-            input.action === 'middle_click' ||
-            input.action === 'double_click' ||
-            input.action === 'triple_click' ||
-            input.action === 'left_mouse_down' ||
-            input.action === 'left_mouse_up' ||
-            input.action === 'left_click_drag' ||
-            input.action === 'scroll' ||
-            input.action === 'zoom' ||
-            input.action === 'type' ||
-            input.action === 'key' ||
-            input.action === 'hold_key';
+          const requiresActionLease = isCuMutatingAction(input.action);
           const leaseResult = requiresActionLease ? state.beforeAction() : undefined;
           if (leaseResult && !leaseResult.ok) {
             return sessionFailure(leaseResult.reason);