diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c7595d1d..500c6c144 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -489,9 +489,6 @@ jobs: - name: Integration crate unit tests run: cargo test --manifest-path harness/evals/integration/Cargo.toml - - name: Validate integration scenarios - run: make -C harness integration-validate - - name: Run integration scenarios run: make -C harness integration-e2e III_BIN="${{ steps.engine.outputs.bin }}" INTEGRATION_REPEAT=2 diff --git a/console/web/e2e/durable-hydration.spec.ts b/console/web/e2e/durable-hydration.spec.ts new file mode 100644 index 000000000..126cb083c --- /dev/null +++ b/console/web/e2e/durable-hydration.spec.ts @@ -0,0 +1,44 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +test.use({ scenario: 'streamed-text' }) + +test('hydrates a durable transcript again after a page reload', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await stack.trigger('harness::send', stack.ready.send) + expect(await completed).toMatchObject({ status: 'completed' }) + + await openSession(page, stack.ready) + await expect( + page.locator('[data-message-role="user"]', { + hasText: stack.ready.message, + }), + ).toHaveCount(1) + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'fixture complete', + }), + ).toHaveCount(1) + + await page.reload() + await page + .getByRole('button', { + name: `open ${stack.ready.session.title}`, + exact: true, + }) + .click() + await expect( + page.locator('[data-message-role="user"]', { + hasText: stack.ready.message, + }), + ).toHaveCount(1) + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'fixture complete', + }), + ).toHaveCount(1) + + expectPassingResult(await stack.finish()) +}) diff --git a/console/web/e2e/exactly-once-function.spec.ts b/console/web/e2e/exactly-once-function.spec.ts new file mode 100644 index 000000000..0832cb581 --- /dev/null +++ b/console/web/e2e/exactly-once-function.spec.ts @@ -0,0 +1,34 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +test.use({ scenario: 'exactly-once-function' }) + +test('renders one completed function call and its durable result', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await stack.trigger('harness::send', stack.ready.send) + expect(await completed).toMatchObject({ status: 'completed' }) + + await openSession(page, stack.ready) + const functionId = stack.ready.functions.record + expect(functionId).toBeTruthy() + const card = page.locator('[data-message-role="function-call"]', { + hasText: functionId, + }) + await expect(card).toHaveCount(1) + await expect(card).toHaveAttribute('data-function-id', functionId) + await expect(card).toHaveAttribute('data-function-status', 'done') + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'recorded once', + }), + ).toHaveCount(1) + + const result = await stack.finish() + expectPassingResult(result) + const recordCalls = (result.evidence?.recorder_events ?? []).filter( + (event) => event.kind === 'target_call' && event.function_id === functionId, + ) + expect(recordCalls).toHaveLength(1) +}) diff --git a/console/web/e2e/harness-stack.ts b/console/web/e2e/harness-stack.ts new file mode 100644 index 000000000..dfd0ef6d0 --- /dev/null +++ b/console/web/e2e/harness-stack.ts @@ -0,0 +1,327 @@ +import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, watch } from 'node:fs/promises' +import path from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import type { Page } from '@playwright/test' +import { test as base, expect } from '@playwright/test' +import { type ISdk, registerWorker } from 'iii-browser-sdk' + +interface ReadyManifest { + schema_version: '1' + run_id: string + scenario_id: string + scenario_slug: string + driver: 'direct' | 'console' + run_root: string + result_path: string + console_url: string + engine_url: string + session: { id: string; title: string } + model: { id: string; provider: string } + message: string + functions: Record + send?: Record +} + +export interface RecorderEvent { + schema_version: '1' + run_id: string + sequence: number + kind: 'target_call' | 'lifecycle' + function_id: string + payload: unknown + received_at: string +} + +/** Raw serialized RunEvidence: real ids, checkable against ReadyManifest. */ +export interface RunEvidence { + run_id: string + session_id: string + turn_id: string | null + send_response: unknown + status: unknown + transcript: unknown[] + generations_consumed: number + generations_total: number + recorder_events: RecorderEvent[] +} + +export interface ServeResult { + schema_version: '1' + scenario_id: string + classification: + | 'pass' + | 'setup_error' + | 'contract_failure' + | 'timeout' + | 'process_crash' + | 'runner_error' + failure: string | null + evidence: RunEvidence | null + artifacts: string[] +} + +interface TurnCompletedEvent { + session_id: string + turn_id: string + status: 'completed' | 'cancelled' | 'failed' + timestamp: number +} + +export interface HarnessStack { + ready: ReadyManifest + trigger(functionId: string, payload: unknown): Promise + waitForTurnCompleted(): Promise + finish(): Promise +} + +interface FixtureOptions { + scenario: string +} + +interface FixtureValues { + stack: HarnessStack +} + +function required(name: string): string { + const value = process.env[name] + if (!value) throw new Error(`${name} is required for Console E2E`) + return value +} + +function workerArgs(): string[] { + return [ + ['queue', 'QUEUE_BIN'], + ['iii-directory', 'III_DIRECTORY_BIN'], + ['session-manager', 'SESSION_MANAGER_BIN'], + ['context-manager', 'CONTEXT_MANAGER_BIN'], + ].flatMap(([name, env]) => ['--worker-bin', `${name}=${required(env)}`]) +} + +async function waitForReady( + readyFile: string, + childExit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, +): Promise { + const read = async (): Promise => { + try { + return JSON.parse(await readFile(readyFile, 'utf8')) as ReadyManifest + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ENOENT' || error instanceof SyntaxError) return null + throw error + } + } + const existing = await read() + if (existing) return existing + + const parent = path.dirname(readyFile) + const expectedName = path.basename(readyFile) + const changes = watch(parent) + const timeout = delay(70_000).then(() => { + throw new Error(`timed out waiting for ${readyFile}`) + }) + const exited = childExit.then(({ code, signal }) => { + throw new Error( + `harness-integration exited before ready (code=${String(code)}, signal=${String(signal)})`, + ) + }) + const appeared = (async () => { + for await (const event of changes) { + if (event.filename && event.filename !== expectedName) continue + const manifest = await read() + if (manifest) return manifest + } + throw new Error(`ready-file watcher closed before ${readyFile} appeared`) + })() + try { + return await Promise.race([appeared, exited, timeout]) + } finally { + await changes.return?.() + } +} + +function childExit( + child: ChildProcessWithoutNullStreams, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + return new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })) + }) +} + +function armCompletion( + sdk: ISdk, + ready: ReadyManifest, +): Promise { + const functionId = `console-e2e::turn-completed::${ready.run_id}` + let functionRef: ReturnType | undefined + let triggerRef: ReturnType | undefined + let timer: NodeJS.Timeout | undefined + const cleanup = () => { + if (timer) clearTimeout(timer) + try { + triggerRef?.unregister() + } catch { + // The stack may already be shutting down. + } + try { + functionRef?.unregister() + } catch { + // The stack may already be shutting down. + } + } + const completed = new Promise((resolve, reject) => { + functionRef = sdk.registerFunction( + functionId, + async (payload) => { + const event = payload as TurnCompletedEvent + if (event.session_id !== ready.session.id) return null + cleanup() + resolve(event) + return null + }, + { metadata: { internal: true } }, + ) + triggerRef = sdk.registerTrigger({ + type: 'harness::turn-completed', + function_id: functionId, + config: { session_id: ready.session.id }, + }) + timer = setTimeout(() => { + cleanup() + reject(new Error('harness::turn-completed was not delivered')) + }, 60_000) + }) + return completed +} + +export const test = base.extend({ + scenario: ['', { scope: 'worker', option: true }], + stack: async ({ scenario }, use, testInfo) => { + if (!scenario) throw new Error('test.use({ scenario }) is required') + const artifactsRoot = path.resolve( + process.env.CONSOLE_E2E_ARTIFACTS_DIR ?? + path.join(testInfo.project.outputDir, '..'), + ) + await mkdir(artifactsRoot, { recursive: true }) + const controlDir = await mkdtemp(path.join(artifactsRoot, 'runner-')) + const readyFile = path.join(controlDir, 'ready.json') + const args = [ + 'serve', + '--scenario', + scenario, + '--engine-bin', + required('III_BIN'), + '--harness-bin', + required('HARNESS_BIN'), + '--console-bin', + required('CONSOLE_BIN'), + '--artifacts-dir', + artifactsRoot, + '--ready-file', + readyFile, + ...workerArgs(), + ] + const child = spawn(required('HARNESS_INTEGRATION_BIN'), args, { + stdio: ['pipe', 'pipe', 'pipe'], + }) + const exit = childExit(child) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)) + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)) + + let sdk: ISdk | undefined + let finalized: Promise | undefined + const finish = (): Promise => { + if (finalized) return finalized + finalized = (async () => { + if (sdk) await sdk.shutdown().catch(() => undefined) + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGTERM') + } + let exited = await Promise.race([exit, delay(30_000).then(() => null)]) + if (!exited) { + child.kill('SIGKILL') + exited = await exit + } + await testInfo.attach('harness-integration.stdout', { + body: Buffer.concat(stdout), + contentType: 'text/plain', + }) + await testInfo.attach('harness-integration.stderr', { + body: Buffer.concat(stderr), + contentType: 'text/plain', + }) + const result = JSON.parse( + await readFile(ready.result_path, 'utf8'), + ) as ServeResult + await testInfo.attach('serve-result', { + body: JSON.stringify(result, null, 2), + contentType: 'application/json', + }) + return result + })() + return finalized + } + + let ready!: ReadyManifest + try { + ready = await waitForReady(readyFile, exit) + } catch (error) { + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGTERM') + } + await exit.catch(() => undefined) + await testInfo.attach('harness-integration.stdout', { + body: Buffer.concat(stdout), + contentType: 'text/plain', + }) + await testInfo.attach('harness-integration.stderr', { + body: Buffer.concat(stderr), + contentType: 'text/plain', + }) + throw error + } + const connectedSdk = registerWorker(ready.engine_url) + sdk = connectedSdk + const stack: HarnessStack = { + ready, + trigger: (functionId: string, payload: unknown) => + connectedSdk.trigger({ + function_id: functionId, + payload, + timeoutMs: 30_000, + }), + waitForTurnCompleted: () => armCompletion(connectedSdk, ready), + finish, + } + + try { + await use(stack) + } finally { + if (!finalized) { + await finish().catch(() => undefined) + } + await rm(controlDir, { recursive: true, force: true }) + } + }, +}) + +export { expect } + +export async function openSession( + page: Page, + ready: ReadyManifest, +): Promise { + await page.goto(ready.console_url) + const session = page.getByRole('button', { + name: `open ${ready.session.title}`, + exact: true, + }) + await session.click() + await expect(session).toHaveAttribute('aria-current', 'page') +} + +export function expectPassingResult(result: ServeResult): void { + expect(result.classification).toBe('pass') +} diff --git a/console/web/e2e/ui-send.spec.ts b/console/web/e2e/ui-send.spec.ts new file mode 100644 index 000000000..5c1405dc4 --- /dev/null +++ b/console/web/e2e/ui-send.spec.ts @@ -0,0 +1,33 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +test.use({ scenario: 'console-streamed-text' }) + +test('sends from the production Console and renders streamed text', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await openSession(page, stack.ready) + + await page + .getByRole('textbox', { name: 'message composer' }) + .fill(stack.ready.message) + await page.getByRole('button', { name: 'send message' }).click() + + await expect( + page.locator('[data-message-role="user"]', { + hasText: stack.ready.message, + }), + ).toHaveCount(1) + expect(await completed).toMatchObject({ + session_id: stack.ready.session.id, + status: 'completed', + }) + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'console fixture complete', + }), + ).toHaveCount(1) + + expectPassingResult(await stack.finish()) +}) diff --git a/console/web/package.json b/console/web/package.json index bf7cbdd6b..2466dce68 100644 --- a/console/web/package.json +++ b/console/web/package.json @@ -3,16 +3,19 @@ "private": true, "version": "0.0.0", "type": "module", - "packageManager": "pnpm@10.18.2", + "packageManager": "pnpm@11.13.1", "scripts": { "dev": "vite", "build": "tsc -b && vite build", "preview": "vite preview", "typecheck": "tsc -b --noEmit", + "typecheck:e2e": "tsc -p tsconfig.e2e.json", "lint": "biome check .", "lint:fix": "biome check --write .", "test": "vitest run", "test:watch": "vitest", + "test:e2e": "playwright test", + "test:e2e:install": "playwright install chromium", "storybook": "storybook dev -p 6006", "build-storybook": "storybook build" }, @@ -45,6 +48,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", + "@playwright/test": "^1.61.1", "@storybook/addon-a11y": "^10.4.1", "@storybook/addon-docs": "^10.4.1", "@storybook/react-vite": "^10.4.1", diff --git a/console/web/playwright.config.ts b/console/web/playwright.config.ts new file mode 100644 index 000000000..7f5c7775d --- /dev/null +++ b/console/web/playwright.config.ts @@ -0,0 +1,23 @@ +import path from 'node:path' +import { defineConfig } from '@playwright/test' + +const artifactsRoot = + process.env.CONSOLE_E2E_ARTIFACTS_DIR ?? + path.resolve(import.meta.dirname, '../../target/console-e2e') + +export default defineConfig({ + testDir: './e2e', + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 120_000, + expect: { timeout: 15_000 }, + reporter: [['list']], + outputDir: path.join(artifactsRoot, 'playwright-output'), + use: { + browserName: 'chromium', + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, +}) diff --git a/console/web/pnpm-lock.yaml b/console/web/pnpm-lock.yaml index 061299d48..2d13033ca 100644 --- a/console/web/pnpm-lock.yaml +++ b/console/web/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: '@biomejs/biome': specifier: ^2.4.15 version: 2.4.15 + '@playwright/test': + specifier: ^1.61.1 + version: 1.61.1 '@storybook/addon-a11y': specifier: ^10.4.1 version: 10.4.1(storybook@10.4.1(@testing-library/dom@10.4.1)(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)) @@ -826,6 +829,11 @@ packages: resolution: {integrity: sha512-sWHv11TMoqKxKDgTIk5VbhQjdPhs8DCcBxbjh3mRlS3YOM/OcrWoGX6MM8eBGn9cUu3M46Py0JnxsG2nJaFTuA==} engines: {vscode: ^1.0.0} + '@playwright/test@1.61.1': + resolution: {integrity: sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==} + engines: {node: '>=18'} + hasBin: true + '@preact/signals-core@1.14.2': resolution: {integrity: sha512-RZHdBj9ZF4n40Rp4jS052EHHjBWf96P9oNdXPfhQTovCuWY9iQn3Gq+gOTJSgBO9A/JBuPfMOWsSX/lIU9Pc/A==} @@ -2124,6 +2132,11 @@ packages: picomatch: optional: true + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -2568,6 +2581,16 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} + playwright-core@1.61.1: + resolution: {integrity: sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.61.1: + resolution: {integrity: sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==} + engines: {node: '>=18'} + hasBin: true + postcss@8.5.14: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} @@ -3648,6 +3671,10 @@ snapshots: '@pierre/theme@1.0.3': {} + '@playwright/test@1.61.1': + dependencies: + playwright: 1.61.1 + '@preact/signals-core@1.14.2': {} '@radix-ui/number@1.1.1': {} @@ -4858,6 +4885,9 @@ snapshots: optionalDependencies: picomatch: 4.0.4 + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -5521,6 +5551,14 @@ snapshots: picomatch@4.0.4: {} + playwright-core@1.61.1: {} + + playwright@1.61.1: + dependencies: + playwright-core: 1.61.1 + optionalDependencies: + fsevents: 2.3.2 + postcss@8.5.14: dependencies: nanoid: 3.3.12 diff --git a/console/web/pnpm-workspace.yaml b/console/web/pnpm-workspace.yaml new file mode 100644 index 000000000..5ed0b5af0 --- /dev/null +++ b/console/web/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + esbuild: true diff --git a/console/web/src/components/chat/LexicalShell.tsx b/console/web/src/components/chat/LexicalShell.tsx index 76082da84..fe4c29fa8 100644 --- a/console/web/src/components/chat/LexicalShell.tsx +++ b/console/web/src/components/chat/LexicalShell.tsx @@ -252,6 +252,7 @@ export function LexicalShell({ diff --git a/console/web/src/components/chat/Message.tsx b/console/web/src/components/chat/Message.tsx index bdf822b13..9dc4901b5 100644 --- a/console/web/src/components/chat/Message.tsx +++ b/console/web/src/components/chat/Message.tsx @@ -266,7 +266,7 @@ function SpawnTaskMessage({ message }: { message: UserMessageType }) { function UserMessage({ message }: { message: UserMessageType }) { return ( -
+
you
@@ -292,7 +292,7 @@ function UserMessage({ message }: { message: UserMessageType }) { function AssistantMessage({ message }: { message: AssistantMessageType }) { const showCaret = !!message.streaming return ( -
+
assistant {message.model ? ( diff --git a/console/web/src/components/function-call/FunctionCallCard.tsx b/console/web/src/components/function-call/FunctionCallCard.tsx index e19c98eaf..a66445d20 100644 --- a/console/web/src/components/function-call/FunctionCallCard.tsx +++ b/console/web/src/components/function-call/FunctionCallCard.tsx @@ -10,11 +10,11 @@ import { DirectoryToolView, } from '@/components/chat/directory' import { EngineFunctionIdLabel, EngineToolView } from '@/components/chat/engine' +import { FpFunctionIdLabel, FpToolView } from '@/components/chat/fp' import { HarnessFunctionIdLabel, HarnessToolView, } from '@/components/chat/harness' -import { FpFunctionIdLabel, FpToolView } from '@/components/chat/fp' import { RouterFunctionIdLabel, RouterToolView } from '@/components/chat/router' import { SandboxFunctionIdLabel, @@ -351,6 +351,11 @@ export function FunctionCallCard({ !embedded && 'border border-rule bg-bg', )} data-message-id={message.id} + data-message-role="function-call" + data-function-id={message.functionId} + data-function-status={ + pending ? 'pending' : running ? 'running' : errored ? 'error' : 'done' + } >