feat(cu): add fenced OpenAI computer observation loop

packages/runtime/src/__tests__/openai-computer-actions.test.ts

@@ -0,0 +1,94 @@
+import assert from 'node:assert/strict';
+import { describe, test } from 'node:test';
+import {
+  convertOpenAIComputerAction,
+  isOpenAIComputerActionSafeByDefault,
+} from '../openai-computer-actions.js';
+
+describe('convertOpenAIComputerAction', () => {
+  test('defaults to observation-only actions under the physical-input safety policy', () => {
+    assert.equal(isOpenAIComputerActionSafeByDefault({ type: 'screenshot' }), true);
+    assert.equal(isOpenAIComputerActionSafeByDefault({ type: 'wait' }), true);
+    assert.equal(isOpenAIComputerActionSafeByDefault({ type: 'move', x: 1, y: 2 }), true);
+    assert.equal(isOpenAIComputerActionSafeByDefault({
+      type: 'click', button: 'left', x: 1, y: 2,
+    }), false);
+    assert.equal(isOpenAIComputerActionSafeByDefault({ type: 'type', text: 'x' }), false);
+    assert.equal(isOpenAIComputerActionSafeByDefault({
+      type: 'keypress', keys: ['ENTER'],
+    }), false);
+  });
+
+  test('converts lossless pointer, keyboard, type, wait, and screenshot actions', () => {
+    assert.deepEqual(convertOpenAIComputerAction({
+      type: 'click', button: 'right', x: 10, y: 20,
+    }), {
+      ok: true,
+      actions: [{ type: 'right_click', coordinate: { x: 10, y: 20 } }],
+    });
+    assert.deepEqual(convertOpenAIComputerAction({
+      type: 'keypress', keys: ['ENTER'],
+    }), {
+      ok: true,
+      actions: [{ type: 'key', text: 'ENTER' }],
+    });
+    assert.deepEqual(convertOpenAIComputerAction({ type: 'type', text: 'hello' }), {
+      ok: true,
+      actions: [{ type: 'type', text: 'hello' }],
+    });
+    assert.deepEqual(convertOpenAIComputerAction({ type: 'wait' }), {
+      ok: true,
+      actions: [{ type: 'wait', durationMs: 2000 }],
+    });
+    assert.deepEqual(convertOpenAIComputerAction({ type: 'screenshot' }), {
+      ok: true,
+      actions: [{ type: 'screenshot' }],
+    });
+  });
+
+  test('converts only a two-point drag path', () => {
+    assert.deepEqual(convertOpenAIComputerAction({
+      type: 'drag',
+      path: [{ x: 1, y: 2 }, { x: 3, y: 4 }],
+    }), {
+      ok: true,
+      actions: [{
+        type: 'left_click_drag',
+        startCoordinate: { x: 1, y: 2 },
+        coordinate: { x: 3, y: 4 },
+      }],
+    });
+    const result = convertOpenAIComputerAction({
+      type: 'drag',
+      path: [{ x: 1, y: 2 }, { x: 2, y: 3 }, { x: 3, y: 4 }],
+    });
+    assert.equal(result.ok, false);
+    if (!result.ok) assert.equal(result.code, 'unsupported_drag_path');
+  });
+
+  test('fails closed for pixel scroll deltas, held modifiers, and navigation buttons', () => {
+    const scroll = convertOpenAIComputerAction({
+      type: 'scroll', x: 1, y: 2, scroll_x: 0, scroll_y: 300,
+    });
+    assert.equal(scroll.ok, false);
+    if (!scroll.ok) assert.equal(scroll.code, 'unsupported_scroll_delta');
+
+    const modified = convertOpenAIComputerAction({
+      type: 'click', button: 'left', x: 1, y: 2, keys: ['SHIFT'],
+    });
+    assert.equal(modified.ok, false);
+    if (!modified.ok) assert.equal(modified.code, 'unsupported_modifier_keys');
+
+    const back = convertOpenAIComputerAction({
+      type: 'click', button: 'back', x: 1, y: 2,
+    });
+    assert.equal(back.ok, false);
+    if (!back.ok) assert.equal(back.code, 'unsupported_button');
+
+    const chord = convertOpenAIComputerAction({
+      type: 'keypress', keys: ['CTRL', 'L'],
+    });
+    assert.equal(chord.ok, false);
+    if (!chord.ok) assert.equal(chord.code, 'unsupported_keypress_chord');
+  });
+});

packages/runtime/src/__tests__/openai-computer-codec.test.ts

@@ -0,0 +1,142 @@
+import assert from 'node:assert/strict';
+import { describe, test } from 'node:test';
+import {
+  createOpenAIComputerContinuationRequest,
+  createOpenAIComputerInitialRequest,
+  decodeOpenAIComputerResponse,
+} from '../openai-computer-codec.js';
+import { OPENAI_COMPUTER_INSTRUCTIONS } from '../openai-computer-policy.js';
+
+const common = {
+  type: 'computer_call',
+  id: 'item_1',
+  call_id: 'call_1',
+  status: 'completed',
+  pending_safety_checks: [],
+} as const;
+const responseBase = { status: 'completed', error: null } as const;
+
+describe('OpenAI computer codec', () => {
+  test('decodes strict GA actions[] and strict preview action shapes', () => {
+    const ga = decodeOpenAIComputerResponse({
+      id: 'resp_1', ...responseBase,
+      output: [{ ...common, actions: [{ type: 'screenshot' }] }],
+    }, 'ga');
+    assert.deepEqual(ga.calls[0].actions, [{ type: 'screenshot' }]);
+
+    const preview = decodeOpenAIComputerResponse({
+      id: 'resp_2', ...responseBase,
+      output: [{ ...common, action: { type: 'wait' } }],
+    }, 'preview');
+    assert.deepEqual(preview.calls[0].actions, [{ type: 'wait' }]);
+
+    const omittedSafety = decodeOpenAIComputerResponse({
+      id: 'resp_3',
+      ...responseBase,
+      output: [{
+        type: 'computer_call',
+        id: 'item_3',
+        call_id: 'call_3',
+        status: 'completed',
+        actions: [{ type: 'screenshot' }],
+      }],
+    }, 'ga');
+    assert.deepEqual(omittedSafety.calls[0].pendingSafetyChecks, []);
+
+    const terminalText = decodeOpenAIComputerResponse({
+      id: 'resp_4',
+      ...responseBase,
+      output: [{
+        type: 'message',
+        content: [
+          { type: 'output_text', text: 'final ' },
+          { type: 'output_text', text: 'answer' },
+        ],
+      }],
+    }, 'ga');
+    assert.equal(terminalText.text, 'final answer');
+  });
+
+  test('rejects mixed dialects and unknown action fields', () => {
+    assert.throws(() => decodeOpenAIComputerResponse({
+      id: 'resp_1', ...responseBase,
+      output: [{ ...common, action: { type: 'wait' } }],
+    }, 'ga'));
+    assert.throws(() => decodeOpenAIComputerResponse({
+      id: 'resp_1', ...responseBase,
+      output: [{
+        ...common,
+        actions: [{ type: 'click', button: 'left', x: 1, y: 2, keys: null, ignored: true }],
+      }],
+    }, 'ga'));
+  });
+
+  test('encodes GA and preview requests without conflating tool contracts', () => {
+    assert.deepEqual(createOpenAIComputerInitialRequest({
+      dialect: 'ga',
+      model: 'gpt',
+      prompt: 'go',
+    }), {
+      model: 'gpt',
+      instructions: OPENAI_COMPUTER_INSTRUCTIONS,
+      tools: [{ type: 'computer' }],
+      input: 'go',
+      parallel_tool_calls: false,
+    });
+    assert.deepEqual(createOpenAIComputerInitialRequest({
+      dialect: 'preview',
+      model: 'computer-use-preview',
+      prompt: 'go',
+      display: { widthPx: 1024, heightPx: 768, environment: 'browser' },
+    }), {
+      model: 'computer-use-preview',
+      instructions: OPENAI_COMPUTER_INSTRUCTIONS,
+      tools: [{
+        type: 'computer_use_preview',
+        display_width: 1024,
+        display_height: 768,
+        environment: 'browser',
+      }],
+      input: 'go',
+      truncation: 'auto',
+      parallel_tool_calls: false,
+    });
+  });
+
+  test('rejects empty GA actions and preserves terminal response failures', () => {
+    assert.throws(() => decodeOpenAIComputerResponse({
+      id: 'resp_empty',
+      ...responseBase,
+      output: [{ ...common, actions: [] }],
+    }, 'ga'));
+    const failed = decodeOpenAIComputerResponse({
+      id: 'resp_failed',
+      status: 'failed',
+      error: { type: 'server_error', code: 'capacity', message: 'No capacity' },
+      output: [],
+    }, 'ga');
+    assert.equal(failed.status, 'failed');
+    assert.equal(failed.error?.code, 'capacity');
+  });
+
+  test('encodes screenshot continuation and explicit safety acknowledgements', () => {
+    const request = createOpenAIComputerContinuationRequest({
+      dialect: 'ga',
+      model: 'gpt',
+      previousResponseId: 'resp_1',
+      callId: 'call_1',
+      screenshot: { base64: 'AA==', mimeType: 'image/png' },
+      acknowledgedSafetyChecks: [{ id: 'safe_1', code: 'x', message: 'confirm' }],
+    });
+    assert.deepEqual(request.input, [{
+      type: 'computer_call_output',
+      call_id: 'call_1',
+      output: {
+        type: 'computer_screenshot',
+        image_url: 'data:image/png;base64,AA==',
+        detail: 'original',
+      },
+      acknowledged_safety_checks: [{ id: 'safe_1', code: 'x', message: 'confirm' }],
+    }]);
+  });
+});