From b4a5e071ec31a47c7ebd9ee0365378bdb78de9c0 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Wed, 22 Jul 2026 00:40:53 -0300 Subject: [PATCH 1/5] (MOT-4107) refactor(integration): simplify boot readiness and add the observe driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Boot readiness moves from structural contract probing (readiness/, ~850 lines: golden schemas, config-entry diffing, queue-topic checks) to presence-only discovery (discovery.rs, ~90 lines): wait for the harness function surface plus the workers it calls mid-turn to be registered, nothing more. Completion is now event-driven — Arm binds harness::turn-completed once and Await blocks on delivery instead of polling harness::status every 250ms, with one status call afterward as the durable-state confirmation the floor checks. Replaces the Console/serve driver with Observe: the integration keeps owning stimulus (harness::send after a start.json signal) and Playwright owns the Console process and DOM assertions directly, spawning CONSOLE_BIN itself instead of asking the integration binary to do it. --console-bin, spawn_console, and the HTTP-port plumbing are gone from the crate; ObserveReadyV1/ObserveResultV1 replace the old serve types. Console e2e specs call stack.start() instead of stack.trigger('harness::send', ...). Also: - Router function goldens are read directly from llm-router/tests/golden/schemas/*.json via include_str! instead of a local copy embedded in readiness contracts. - --repeat/INTEGRATION_REPEAT is gone (CLI, Makefile, CI): the byte-stable scrub is covered by tests/determinism.rs, so booting the full stack twice per scenario was a meta-test, not a contract check. - ModelFixtureV1 drops pricing, reasoning_efforts, thinking_budgets, input_limit, and display_name — fields the compiler always left None and nothing downstream (the scripted router or the harness's own Model parse) ever reads. - Recorder::snapshot() drops its unused after_sequence filter; a few internal-only helpers (Client::call_with_timeout, Deadline::at/cap, the stack config re-exports) move to pub(crate) or private; readiness_deadline is renamed setup_deadline now that the module it named is gone. No scenario behavior change. Validated with cargo fmt/clippy (clean), cargo test (96 passing), and live Direct-driver runs (E2E-001, E2E-002) against a local stack. --- .github/workflows/ci.yml | 2 +- console/web/e2e/durable-hydration.spec.ts | 4 +- console/web/e2e/exactly-once-function.spec.ts | 4 +- console/web/e2e/harness-stack.ts | 173 +++++- console/web/e2e/ui-send.spec.ts | 10 +- harness/Makefile | 2 - harness/evals/integration/README.md | 36 +- harness/evals/integration/src/artifacts.rs | 5 +- .../evals/integration/src/artifacts/sink.rs | 2 +- harness/evals/integration/src/canonical.rs | 8 +- harness/evals/integration/src/client.rs | 7 +- harness/evals/integration/src/deadline.rs | 5 +- harness/evals/integration/src/discovery.rs | 96 ++++ .../evals/integration/src/evidence_data.rs | 5 +- harness/evals/integration/src/expand.rs | 5 - .../evals/integration/src/expand/router.rs | 1 - .../src/fixtures/script_validation.rs | 14 +- .../evals/integration/src/fixtures/tests.rs | 4 +- harness/evals/integration/src/lib.rs | 21 +- harness/evals/integration/src/main.rs | 175 ++---- harness/evals/integration/src/matcher.rs | 26 - harness/evals/integration/src/process.rs | 8 +- harness/evals/integration/src/process/spec.rs | 14 - .../integration/src/process/supervisor.rs | 4 - harness/evals/integration/src/readiness.rs | 24 - .../integration/src/readiness/catalog.rs | 176 ------ .../integration/src/readiness/contracts.rs | 396 -------------- .../evals/integration/src/readiness/probe.rs | 201 ------- .../evals/integration/src/readiness/spec.rs | 53 -- harness/evals/integration/src/recorder.rs | 1 - .../evals/integration/src/recorder/service.rs | 10 +- .../evals/integration/src/recorder/store.rs | 13 +- .../evals/integration/src/recorder/tests.rs | 16 +- harness/evals/integration/src/scenario.rs | 8 +- .../evals/integration/src/scenario/floor.rs | 8 +- .../evals/integration/src/scenario/observe.rs | 391 ++++++++++++++ .../integration/src/scenario/phases/arm.rs | 95 ++++ .../src/scenario/phases/completion.rs | 122 +---- .../src/scenario/phases/evidence.rs | 4 +- .../src/scenario/phases/execution.rs | 155 +++--- .../integration/src/scenario/phases/mod.rs | 66 +-- .../src/scenario/phases/readiness.rs | 155 ------ .../evals/integration/src/scenario/runner.rs | 135 +++-- .../evals/integration/src/scenario/serve.rs | 507 ------------------ .../evals/integration/src/scenario/state.rs | 6 +- .../integration/src/scenarios/builder.rs | 14 - .../src/scenarios/console_streamed_text.rs | 21 +- .../evals/integration/src/scenarios/mod.rs | 8 +- .../evals/integration/src/scripted_router.rs | 91 +++- harness/evals/integration/src/stack.rs | 7 - harness/evals/integration/src/stack/bins.rs | 5 +- harness/evals/integration/src/stack/config.rs | 13 - harness/evals/integration/src/stack/layout.rs | 3 - .../evals/integration/src/stack/manifest.rs | 8 +- .../evals/integration/src/stack/supervisor.rs | 19 - harness/evals/integration/src/stack/tests.rs | 3 - .../integration/src/types/scenario/result.rs | 4 +- harness/evals/integration/src/types/script.rs | 35 -- .../evals/integration/tests/determinism.rs | 6 +- harness/evals/integration/tests/readiness.rs | 171 ------ harness/evals/integration/tests/supervisor.rs | 6 +- 61 files changed, 1179 insertions(+), 2408 deletions(-) create mode 100644 harness/evals/integration/src/discovery.rs delete mode 100644 harness/evals/integration/src/readiness.rs delete mode 100644 harness/evals/integration/src/readiness/catalog.rs delete mode 100644 harness/evals/integration/src/readiness/contracts.rs delete mode 100644 harness/evals/integration/src/readiness/probe.rs delete mode 100644 harness/evals/integration/src/readiness/spec.rs create mode 100644 harness/evals/integration/src/scenario/observe.rs create mode 100644 harness/evals/integration/src/scenario/phases/arm.rs delete mode 100644 harness/evals/integration/src/scenario/phases/readiness.rs delete mode 100644 harness/evals/integration/src/scenario/serve.rs delete mode 100644 harness/evals/integration/tests/readiness.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c7595d1d..73942324d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -493,7 +493,7 @@ jobs: run: make -C harness integration-validate - name: Run integration scenarios - run: make -C harness integration-e2e III_BIN="${{ steps.engine.outputs.bin }}" INTEGRATION_REPEAT=2 + run: make -C harness integration-e2e III_BIN="${{ steps.engine.outputs.bin }}" - name: Verify integration report links run: | diff --git a/console/web/e2e/durable-hydration.spec.ts b/console/web/e2e/durable-hydration.spec.ts index 126cb083c..5b7829cb0 100644 --- a/console/web/e2e/durable-hydration.spec.ts +++ b/console/web/e2e/durable-hydration.spec.ts @@ -7,10 +7,10 @@ test('hydrates a durable transcript again after a page reload', async ({ stack, }) => { const completed = stack.waitForTurnCompleted() - await stack.trigger('harness::send', stack.ready.send) + await stack.start() expect(await completed).toMatchObject({ status: 'completed' }) - await openSession(page, stack.ready) + await openSession(page, stack) await expect( page.locator('[data-message-role="user"]', { hasText: stack.ready.message, diff --git a/console/web/e2e/exactly-once-function.spec.ts b/console/web/e2e/exactly-once-function.spec.ts index 0832cb581..6e56b62c9 100644 --- a/console/web/e2e/exactly-once-function.spec.ts +++ b/console/web/e2e/exactly-once-function.spec.ts @@ -7,10 +7,10 @@ test('renders one completed function call and its durable result', async ({ stack, }) => { const completed = stack.waitForTurnCompleted() - await stack.trigger('harness::send', stack.ready.send) + await stack.start() expect(await completed).toMatchObject({ status: 'completed' }) - await openSession(page, stack.ready) + await openSession(page, stack) const functionId = stack.ready.functions.record expect(functionId).toBeTruthy() const card = page.locator('[data-message-role="function-call"]', { diff --git a/console/web/e2e/harness-stack.ts b/console/web/e2e/harness-stack.ts index dfd0ef6d0..54b0b2d57 100644 --- a/console/web/e2e/harness-stack.ts +++ b/console/web/e2e/harness-stack.ts @@ -1,5 +1,6 @@ -import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process' -import { mkdir, mkdtemp, readFile, rm, watch } from 'node:fs/promises' +import { type ChildProcess, spawn } from 'node:child_process' +import { createServer } from 'node:net' +import { mkdir, mkdtemp, readFile, rename, rm, writeFile, watch } from 'node:fs/promises' import path from 'node:path' import { setTimeout as delay } from 'node:timers/promises' import type { Page } from '@playwright/test' @@ -11,16 +12,15 @@ interface ReadyManifest { run_id: string scenario_id: string scenario_slug: string - driver: 'direct' | 'console' + driver: 'direct' | 'observe' run_root: string result_path: string - console_url: string engine_url: string session: { id: string; title: string } model: { id: string; provider: string } message: string functions: Record - send?: Record + send: Record } export interface RecorderEvent { @@ -46,7 +46,7 @@ export interface RunEvidence { recorder_events: RecorderEvent[] } -export interface ServeResult { +export interface ObserveResult { schema_version: '1' scenario_id: string classification: @@ -70,9 +70,10 @@ interface TurnCompletedEvent { export interface HarnessStack { ready: ReadyManifest - trigger(functionId: string, payload: unknown): Promise + consoleUrl: string + start(): Promise waitForTurnCompleted(): Promise - finish(): Promise + finish(): Promise } interface FixtureOptions { @@ -98,6 +99,36 @@ function workerArgs(): string[] { ].flatMap(([name, env]) => ['--worker-bin', `${name}=${required(env)}`]) } +async function freeLoopbackPort(): Promise { + return await new Promise((resolve, reject) => { + const server = createServer() + server.listen(0, '127.0.0.1', () => { + const address = server.address() + if (!address || typeof address === 'string') { + server.close() + reject(new Error('failed to allocate loopback port')) + return + } + const { port } = address + server.close((error) => { + if (error) reject(error) + else resolve(port) + }) + }) + server.on('error', reject) + }) +} + +async function writeAtomicJson(filePath: string, value: unknown): Promise { + const parent = path.dirname(filePath) + const temporary = path.join( + parent, + `.${path.basename(filePath)}.${process.pid}.tmp`, + ) + await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, 'utf8') + await rename(temporary, filePath) +} + async function waitForReady( readyFile: string, childExit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, @@ -140,8 +171,26 @@ async function waitForReady( } } +async function waitForConsoleHttp(port: number): Promise { + const deadline = Date.now() + 60_000 + while (Date.now() < deadline) { + try { + const response = await fetch(`http://127.0.0.1:${port}/`, { + redirect: 'manual', + }) + if (response.ok || (response.status >= 300 && response.status < 400)) { + return + } + } catch { + // Console still booting. + } + await delay(100) + } + throw new Error(`console HTTP did not become ready on port ${port}`) +} + function childExit( - child: ChildProcessWithoutNullStreams, + child: ChildProcess, ): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { return new Promise((resolve) => { child.once('exit', (code, signal) => resolve({ code, signal })) @@ -194,6 +243,12 @@ function armCompletion( return completed } +function stopChild(child: ChildProcess | undefined): void { + if (!child) return + if (child.exitCode !== null || child.signalCode !== null) return + child.kill('SIGTERM') +} + export const test = base.extend({ scenario: ['', { scope: 'worker', option: true }], stack: async ({ scenario }, use, testInfo) => { @@ -205,16 +260,15 @@ export const test = base.extend({ await mkdir(artifactsRoot, { recursive: true }) const controlDir = await mkdtemp(path.join(artifactsRoot, 'runner-')) const readyFile = path.join(controlDir, 'ready.json') + const startFile = path.join(controlDir, 'start.json') const args = [ - 'serve', + 'observe', '--scenario', scenario, '--engine-bin', required('III_BIN'), '--harness-bin', required('HARNESS_BIN'), - '--console-bin', - required('CONSOLE_BIN'), '--artifacts-dir', artifactsRoot, '--ready-file', @@ -231,11 +285,30 @@ export const test = base.extend({ child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)) let sdk: ISdk | undefined - let finalized: Promise | undefined - const finish = (): Promise => { + let consoleChild: ChildProcess | undefined + let consoleExit: Promise<{ + code: number | null + signal: NodeJS.Signals | null + }> | undefined + const consoleStdout: Buffer[] = [] + const consoleStderr: Buffer[] = [] + let finalized: Promise | undefined + + const finish = (): Promise => { if (finalized) return finalized finalized = (async () => { if (sdk) await sdk.shutdown().catch(() => undefined) + stopChild(consoleChild) + if (consoleExit) { + let consoleExited = await Promise.race([ + consoleExit, + delay(10_000).then(() => null), + ]) + if (!consoleExited && consoleChild) { + consoleChild.kill('SIGKILL') + consoleExited = await consoleExit + } + } if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM') } @@ -252,10 +325,20 @@ export const test = base.extend({ body: Buffer.concat(stderr), contentType: 'text/plain', }) + if (consoleStdout.length > 0 || consoleStderr.length > 0) { + await testInfo.attach('console.stdout', { + body: Buffer.concat(consoleStdout), + contentType: 'text/plain', + }) + await testInfo.attach('console.stderr', { + body: Buffer.concat(consoleStderr), + contentType: 'text/plain', + }) + } const result = JSON.parse( await readFile(ready.result_path, 'utf8'), - ) as ServeResult - await testInfo.attach('serve-result', { + ) as ObserveResult + await testInfo.attach('observe-result', { body: JSON.stringify(result, null, 2), contentType: 'application/json', }) @@ -265,13 +348,39 @@ export const test = base.extend({ } let ready!: ReadyManifest + let consoleUrl!: string try { ready = await waitForReady(readyFile, exit) + const httpPort = await freeLoopbackPort() + consoleUrl = `http://127.0.0.1:${httpPort}` + const spawnedConsole = spawn( + required('CONSOLE_BIN'), + ['--url', ready.engine_url, '--http-port', String(httpPort)], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ) + consoleChild = spawnedConsole + consoleExit = childExit(spawnedConsole) + spawnedConsole.stdout.on('data', (chunk: Buffer) => + consoleStdout.push(chunk), + ) + spawnedConsole.stderr.on('data', (chunk: Buffer) => + consoleStderr.push(chunk), + ) + await Promise.race([ + waitForConsoleHttp(httpPort), + consoleExit.then(({ code, signal }) => { + throw new Error( + `console exited before ready (code=${String(code)}, signal=${String(signal)})`, + ) + }), + ]) } catch (error) { + stopChild(consoleChild) if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM') } await exit.catch(() => undefined) + await consoleExit?.catch(() => undefined) await testInfo.attach('harness-integration.stdout', { body: Buffer.concat(stdout), contentType: 'text/plain', @@ -280,18 +389,30 @@ export const test = base.extend({ body: Buffer.concat(stderr), contentType: 'text/plain', }) + if (consoleStdout.length > 0 || consoleStderr.length > 0) { + await testInfo.attach('console.stdout', { + body: Buffer.concat(consoleStdout), + contentType: 'text/plain', + }) + await testInfo.attach('console.stderr', { + body: Buffer.concat(consoleStderr), + contentType: 'text/plain', + }) + } throw error } + const connectedSdk = registerWorker(ready.engine_url) sdk = connectedSdk + let started = false const stack: HarnessStack = { ready, - trigger: (functionId: string, payload: unknown) => - connectedSdk.trigger({ - function_id: functionId, - payload, - timeoutMs: 30_000, - }), + consoleUrl, + start: async () => { + if (started) return + started = true + await writeAtomicJson(startFile, { schema_version: '1' }) + }, waitForTurnCompleted: () => armCompletion(connectedSdk, ready), finish, } @@ -311,17 +432,17 @@ export { expect } export async function openSession( page: Page, - ready: ReadyManifest, + stack: HarnessStack, ): Promise { - await page.goto(ready.console_url) + await page.goto(stack.consoleUrl) const session = page.getByRole('button', { - name: `open ${ready.session.title}`, + name: `open ${stack.ready.session.title}`, exact: true, }) await session.click() await expect(session).toHaveAttribute('aria-current', 'page') } -export function expectPassingResult(result: ServeResult): void { +export function expectPassingResult(result: ObserveResult): void { expect(result.classification).toBe('pass') } diff --git a/console/web/e2e/ui-send.spec.ts b/console/web/e2e/ui-send.spec.ts index 5c1405dc4..5955d1768 100644 --- a/console/web/e2e/ui-send.spec.ts +++ b/console/web/e2e/ui-send.spec.ts @@ -2,17 +2,13 @@ import { expect, expectPassingResult, openSession, test } from './harness-stack' test.use({ scenario: 'console-streamed-text' }) -test('sends from the production Console and renders streamed text', async ({ +test('renders a harness-started streamed turn in the Console', async ({ page, stack, }) => { const completed = stack.waitForTurnCompleted() - await openSession(page, stack.ready) - - await page - .getByRole('textbox', { name: 'message composer' }) - .fill(stack.ready.message) - await page.getByRole('button', { name: 'send message' }).click() + await openSession(page, stack) + await stack.start() await expect( page.locator('[data-message-role="user"]', { diff --git a/harness/Makefile b/harness/Makefile index f97117aa6..1ba679b3e 100644 --- a/harness/Makefile +++ b/harness/Makefile @@ -319,7 +319,6 @@ INTEGRATION_WORKERS := queue iii-directory session-manager context-manager INTEGRATION_PROFILE ?= release INTEGRATION_FLAG := $(if $(filter release,$(INTEGRATION_PROFILE)),--release,) INTEGRATION_SCENARIO ?= all -INTEGRATION_REPEAT ?= 1 INTEGRATION_ARTIFACTS ?= $(REPO_ROOT)/target/integration integration-e2e: @@ -343,7 +342,6 @@ integration-e2e: --worker-bin "session-manager=$(REPO_ROOT)/session-manager/target/$(INTEGRATION_PROFILE)/session-manager" \ --worker-bin "context-manager=$(REPO_ROOT)/context-manager/target/$(INTEGRATION_PROFILE)/context-manager" \ --scenario "$(INTEGRATION_SCENARIO)" \ - --repeat "$(INTEGRATION_REPEAT)" \ --artifacts-dir "$(INTEGRATION_ARTIFACTS)" integration-validate: diff --git a/harness/evals/integration/README.md b/harness/evals/integration/README.md index 12de875ce..14a92ce69 100644 --- a/harness/evals/integration/README.md +++ b/harness/evals/integration/README.md @@ -24,8 +24,7 @@ harness-integration run \ --worker-bin session-manager= \ --worker-bin context-manager= \ --worker-bin iii-directory= \ - --scenario E2E-001 \ - --repeat 2 + --scenario E2E-001 ``` The engine is never downloaded by the runner. CI builds the source revision @@ -38,9 +37,6 @@ Exit codes are: - `2`: contract failure or scenario timeout; - `3`: setup, process, or runner error. -`--repeat N` boots a fresh stack for every repetition and requires the -byte-stable result contract to be identical. A mismatch is a runner error. - ## Create a scenario Each scenario is one Rust module: `src/scenarios/.rs`, one `scenario()` @@ -124,12 +120,11 @@ for execution. Typed text and function-call replies cover normal cases; `.recovery_boundary()` matches a reply against the durable outcome only, where a fault restart or hook release may rebuild the request, and `.match_overrides(...)` is the remaining escape hatch for intentionally -different wire shapes (the Console's agent-trigger policy). +different wire shapes. -Timeout defaults are 60 seconds for readiness, 60 seconds for the scenario, -and 15 seconds for teardown. The scenario budget can be raised with -`.scenario_timeout_ms(...)` (crash-recovery does); one readiness budget is -shared by the full probe/arm sequence. +Timeout defaults are 60 seconds for setup waits (e.g. observer start), 60 +seconds for the scenario, and 15 seconds for teardown. The scenario budget +can be raised with `.scenario_timeout_ms(...)` (crash-recovery does). ## Checked-in scenarios @@ -137,23 +132,26 @@ shared by the full probe/arm sequence. |---|---|---| | E2E-001 | `streamed-text` | streamed text reaches durable completion | | E2E-002 | `exactly-once-function` | a native function executes exactly once | -| UI-001 | `console-streamed-text` | the production Console sends and renders streamed text | +| UI-001 | `console-streamed-text` | integration starts a streamed turn; Playwright validates Console UI | | E2E-505 | `hold-mutation-505` | quarantined reproduction for issue #505 | | E2E-506 | `hook-held-release-506` | quarantined reproduction for issue #506 | | E2E-507 | `crash-recovery-507` | quarantined reproduction for issue #507 | -`run --scenario all` includes non-quarantined direct scenarios. Console-driven -scenarios run through `serve --scenario ` and Playwright. An -explicit quarantined direct scenario still runs; `validate --scenario all` +`run --scenario all` includes non-quarantined direct scenarios. Observe-driven +UI scenarios (and Direct scenarios used from Playwright) run through +`observe --scenario `: the integration publishes `ready.json`, +waits for Playwright's `start.json`, then runs `harness::send` and grades +backend evidence while Playwright owns the Console process and DOM asserts. +An explicit quarantined direct scenario still runs; `validate --scenario all` always includes every driver and quarantine state. ## Runtime and evidence -The lifecycle is allocate → boot → probe → arm → send → optional fault or -release → await → collect → grade → teardown → report. +The lifecycle is allocate → boot → arm → send → optional fault or release → +await → collect → grade → teardown → report. Observe inserts Probe (wait for +`start.json`) between Arm and Send, then waits for observer shutdown after +Await before Collect. -- Readiness inspects structured function, trigger, queue, and configuration - surfaces. - All RPCs and polling share monotonic phase deadlines. - The recorder keeps configuration and snapshots in process; only controlled target functions and the lifecycle sink are registered with the engine. @@ -170,7 +168,7 @@ calls, lifecycle events). `result.json` contains the stable byte-comparable verdict: the classification plus the first floor or verify failure message, with run/session/turn ids scrubbed to placeholders. `execution.json` contains the run id, timing, scenario id, and SHA-256 of the exact `result.json` -bytes. In serve mode, `serve-result.json` additionally carries the raw +bytes. In observe mode, `observe-result.json` additionally carries the raw serialized `RunEvidence` (real ids) so Playwright can check it against the ready manifest. Passing runs retain the compact reports and remove heavyweight stack state unless `--retain-success` is supplied. diff --git a/harness/evals/integration/src/artifacts.rs b/harness/evals/integration/src/artifacts.rs index e3479eef0..22fdebc16 100644 --- a/harness/evals/integration/src/artifacts.rs +++ b/harness/evals/integration/src/artifacts.rs @@ -5,4 +5,7 @@ mod sink; #[cfg(test)] mod tests; -pub use sink::{trim_passing_run, write_json, ArtifactSink}; +pub use sink::{write_json, ArtifactSink}; + +#[cfg(test)] +pub(crate) use sink::trim_passing_run; diff --git a/harness/evals/integration/src/artifacts/sink.rs b/harness/evals/integration/src/artifacts/sink.rs index 14647f66f..a928bff2a 100644 --- a/harness/evals/integration/src/artifacts/sink.rs +++ b/harness/evals/integration/src/artifacts/sink.rs @@ -124,7 +124,7 @@ where /// On a passing run without `--retain-success`, drop heavyweight stack state /// but keep the compact reports and collected evidence. -pub fn trim_passing_run(run_root: &Path) { +pub(crate) fn trim_passing_run(run_root: &Path) { for dir in [ "engine", "logs", diff --git a/harness/evals/integration/src/canonical.rs b/harness/evals/integration/src/canonical.rs index 1a513f85d..76d336769 100644 --- a/harness/evals/integration/src/canonical.rs +++ b/harness/evals/integration/src/canonical.rs @@ -7,7 +7,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; /// A copy of `value` with all object keys sorted recursively. -pub fn sort_keys(value: &Value) -> Value { +fn sort_keys(value: &Value) -> Value { match value { Value::Object(map) => { let mut sorted: Vec<(&String, &Value)> = map.iter().collect(); @@ -37,11 +37,6 @@ pub fn canonical_json_pretty(value: &Value) -> String { text } -/// Lowercase-hex SHA-256 of the canonical JSON of `value`. -pub fn sha256_of_canonical(value: &Value) -> String { - hex(&Sha256::digest(canonical_json(value).as_bytes())) -} - /// Lowercase-hex SHA-256 of raw bytes (e.g. the UTF-8 of a matched string). pub fn sha256_of_bytes(bytes: &[u8]) -> String { hex(&Sha256::digest(bytes)) @@ -61,7 +56,6 @@ mod tests { let a: Value = serde_json::from_str(r#"{"b":1,"a":{"d":2,"c":[{"y":1,"x":2}]}}"#).unwrap(); let b: Value = serde_json::from_str(r#"{"a":{"c":[{"x":2,"y":1}],"d":2},"b":1}"#).unwrap(); assert_eq!(canonical_json(&a), canonical_json(&b)); - assert_eq!(sha256_of_canonical(&a), sha256_of_canonical(&b)); } #[test] diff --git a/harness/evals/integration/src/client.rs b/harness/evals/integration/src/client.rs index d6c95cc24..8f844aa9a 100644 --- a/harness/evals/integration/src/client.rs +++ b/harness/evals/integration/src/client.rs @@ -44,12 +44,7 @@ impl Client { &self.iii } - pub async fn call(&self, function_id: &str, payload: Value) -> Result { - self.call_with_timeout(function_id, payload, DEFAULT_CALL_TIMEOUT_MS) - .await - } - - pub async fn call_with_timeout( + async fn call_with_timeout( &self, function_id: &str, payload: Value, diff --git a/harness/evals/integration/src/deadline.rs b/harness/evals/integration/src/deadline.rs index ce67834e7..4a3dd8831 100644 --- a/harness/evals/integration/src/deadline.rs +++ b/harness/evals/integration/src/deadline.rs @@ -23,7 +23,7 @@ impl Deadline { } /// Use an existing Tokio instant as the deadline. - pub const fn at(expires_at: Instant) -> Self { + const fn at(expires_at: Instant) -> Self { Self { expires_at } } @@ -42,7 +42,7 @@ impl Deadline { } /// Cap an operation-specific timeout by the global deadline. - pub fn cap( + fn cap( self, maximum: Duration, operation: impl Into, @@ -128,6 +128,7 @@ impl DeadlineExceeded { } } + #[cfg(test)] pub fn operation(&self) -> &str { &self.operation } diff --git a/harness/evals/integration/src/discovery.rs b/harness/evals/integration/src/discovery.rs new file mode 100644 index 000000000..f4fbac799 --- /dev/null +++ b/harness/evals/integration/src/discovery.rs @@ -0,0 +1,96 @@ +//! Poll engine discovery until required surfaces appear. This is only a boot +//! race barrier — not schema/contract validation. + +use std::collections::BTreeSet; +use std::time::Duration; + +use serde_json::{json, Value}; + +use crate::client::{Client, DEFAULT_CALL_TIMEOUT_MS}; +use crate::deadline::Deadline; + +const DISCOVERY_POLL_INTERVAL: Duration = Duration::from_millis(200); + +/// Functions that must be registered before a turn can run: the harness +/// entrypoint plus the worker surfaces it calls mid-turn. Router and the +/// lifecycle sink are excluded — `RunServices::start` registers those itself. +pub const TURN_SURFACE: &[&str] = &["harness::send", "session::messages", "context::assemble"]; + +/// Wait until every function id is present in `engine::functions::list`. +pub async fn wait_for_functions( + client: &Client, + function_ids: &[&str], + deadline: Deadline, +) -> anyhow::Result<()> { + let label = format!("functions {}", function_ids.join(", ")); + deadline + .poll_until(label, DISCOVERY_POLL_INTERVAL, || async { + let listed = client + .call_with_deadline( + "engine::functions::list", + json!({ "include_internal": true }), + deadline, + DEFAULT_CALL_TIMEOUT_MS, + ) + .await + .map_err(anyhow::Error::msg)?; + let ids = collect_ids(&listed, &["function_id", "id"]); + if function_ids.iter().all(|id| ids.contains(*id)) { + Ok(Some(())) + } else { + Ok(None) + } + }) + .await +} + +/// Wait until every trigger type is present in `engine::triggers::list`. +pub async fn wait_for_trigger_types( + client: &Client, + trigger_types: &[&str], + deadline: Deadline, +) -> anyhow::Result<()> { + let label = format!("trigger types {}", trigger_types.join(", ")); + deadline + .poll_until(label, DISCOVERY_POLL_INTERVAL, || async { + let listed = client + .call_with_deadline( + "engine::triggers::list", + json!({ "include_internal": true }), + deadline, + DEFAULT_CALL_TIMEOUT_MS, + ) + .await + .map_err(anyhow::Error::msg)?; + let ids = collect_ids(&listed, &["trigger_type", "id", "name", "type"]); + if trigger_types.iter().all(|id| ids.contains(*id)) { + Ok(Some(())) + } else { + Ok(None) + } + }) + .await +} + +fn collect_ids(listed: &Value, keys: &[&str]) -> BTreeSet { + let items: Vec<&Value> = match listed { + Value::Array(items) => items.iter().collect(), + Value::Object(map) => map + .values() + .find_map(|value| value.as_array()) + .map(|items| items.iter().collect()) + .unwrap_or_default(), + _ => Vec::new(), + }; + items + .iter() + .filter_map(|item| { + if let Some(text) = item.as_str() { + return Some(text.to_string()); + } + keys.iter() + .find_map(|key| item.get(key).and_then(Value::as_str)) + .map(String::from) + }) + .collect() +} diff --git a/harness/evals/integration/src/evidence_data.rs b/harness/evals/integration/src/evidence_data.rs index 95f0df103..8b8de701a 100644 --- a/harness/evals/integration/src/evidence_data.rs +++ b/harness/evals/integration/src/evidence_data.rs @@ -16,8 +16,7 @@ pub struct RunEvidence { pub run_id: String, pub session_id: String, pub turn_id: Option, - /// `harness::send` response — `None` in serve mode, where the Console - /// (not the runner) submits the send. + /// `harness::send` response — present after Direct/Observe Send succeeds. pub send_response: Option, /// Final `harness::status` report (JSON null when the session is unknown). pub status: Value, @@ -134,7 +133,7 @@ impl RunEvidence { /// Replace this run's concrete ids with `{{run_id}}` / `{{session_id}}` / /// `{{turn_id}}` placeholders so persisted failure text stays - /// byte-comparable across repetitions. + /// byte-comparable across runs. pub fn scrub(&self, text: &str) -> String { let mut text = text.to_string(); replace_identity(&mut text, &self.run_id, "{{run_id}}"); diff --git a/harness/evals/integration/src/expand.rs b/harness/evals/integration/src/expand.rs index 96628536a..535ffeb7d 100644 --- a/harness/evals/integration/src/expand.rs +++ b/harness/evals/integration/src/expand.rs @@ -161,18 +161,13 @@ fn default_model_fixture() -> ModelFixtureV1 { ModelFixtureV1 { id: DEFAULT_MODEL.to_string(), provider: DEFAULT_PROVIDER.to_string(), - display_name: None, context_window: 32_768, max_output_tokens: 4_096, - input_limit: None, supports_thinking: Some(false), supports_xhigh: None, - reasoning_efforts: None, supports_tools: Some(true), supports_vision: Some(false), supports_cache: Some(false), supports_structured_output: Some(true), - thinking_budgets: None, - pricing: None, } } diff --git a/harness/evals/integration/src/expand/router.rs b/harness/evals/integration/src/expand/router.rs index fe9a645ef..7ab1f7f89 100644 --- a/harness/evals/integration/src/expand/router.rs +++ b/harness/evals/integration/src/expand/router.rs @@ -90,7 +90,6 @@ fn default_match( .map(|index| JsonNormalizerV1 { pointer: format!("/{index}/timestamp"), operation: NormalizerOperation::Delete, - replacement: None, }) .collect(); GenerationMatchV1 { diff --git a/harness/evals/integration/src/fixtures/script_validation.rs b/harness/evals/integration/src/fixtures/script_validation.rs index 4878fa2ff..e2fa49c8d 100644 --- a/harness/evals/integration/src/fixtures/script_validation.rs +++ b/harness/evals/integration/src/fixtures/script_validation.rs @@ -66,22 +66,10 @@ fn validate_matcher(matcher: &JsonMatcherV1) -> anyhow::Result<()> { fn validate_normalizer(normalizer: &JsonNormalizerV1) -> anyhow::Result<()> { crate::matcher::validate_pointer(&normalizer.pointer)?; match normalizer.operation { - NormalizerOperation::Replace if normalizer.replacement.is_none() => { - anyhow::bail!( - "replace normalizer at {:?} requires `replacement`", - normalizer.pointer - ) - } - NormalizerOperation::Delete if normalizer.replacement.is_some() => { - anyhow::bail!( - "delete normalizer at {:?} forbids `replacement`", - normalizer.pointer - ) - } NormalizerOperation::Delete if normalizer.pointer.is_empty() => { anyhow::bail!("delete normalizer cannot target the document root") } - _ => Ok(()), + NormalizerOperation::Delete => Ok(()), } } diff --git a/harness/evals/integration/src/fixtures/tests.rs b/harness/evals/integration/src/fixtures/tests.rs index 166abaccc..77f913b28 100644 --- a/harness/evals/integration/src/fixtures/tests.rs +++ b/harness/evals/integration/src/fixtures/tests.rs @@ -109,13 +109,15 @@ fn invalid_matchers_and_normalizers_are_rejected() { }); assert!(validate(script).is_err()); + // `replace` left the wire contract with its last emitter; the schema now + // rejects it as an unknown operation. let script = minimal_script(|script| { script["generations"][0]["match"]["messages"] = json!({ "mode": "exact", "expected": [], "normalize": [{ "pointer": "/0/x", "operation": "replace" }] }); }); - assert!(error_chain(script).contains("replacement")); + assert!(error_chain(script).contains("replace")); let script = minimal_script(|script| { script["generations"][0]["match"]["messages"] = json!({ diff --git a/harness/evals/integration/src/lib.rs b/harness/evals/integration/src/lib.rs index 9d223fa34..e4a6b3ce2 100644 --- a/harness/evals/integration/src/lib.rs +++ b/harness/evals/integration/src/lib.rs @@ -5,21 +5,22 @@ //! stack, replaces only the `router::*` boundary with a strict scripted //! worker, and verifies structured public evidence. -pub mod artifacts; pub mod canonical; -pub mod client; -pub mod deadline; pub mod evidence_data; pub mod expand; pub mod fixtures; -pub mod matcher; -pub mod process; -pub mod readiness; -pub mod recorder; -pub mod runtime; pub mod scenario; pub mod scenarios; -pub mod scripted_router; -pub mod services; pub mod stack; pub mod types; + +pub(crate) mod artifacts; +pub(crate) mod client; +pub(crate) mod deadline; +pub(crate) mod discovery; +pub(crate) mod matcher; +pub(crate) mod process; +pub(crate) mod recorder; +pub(crate) mod runtime; +pub(crate) mod scripted_router; +pub(crate) mod services; diff --git a/harness/evals/integration/src/main.rs b/harness/evals/integration/src/main.rs index e03aefbc6..2fb5579b8 100644 --- a/harness/evals/integration/src/main.rs +++ b/harness/evals/integration/src/main.rs @@ -5,7 +5,7 @@ use anyhow::Context; use clap::{Args, Parser, Subcommand}; use harness_integration::expand::render_compiled; use harness_integration::fixtures::{scenario_fixtures, ScenarioFixture}; -use harness_integration::scenario::{run_scenario, serve_scenario}; +use harness_integration::scenario::{observe_scenario, run_scenario}; use harness_integration::scenarios::ScenarioDriver; use harness_integration::stack::StackBins; use harness_integration::types::scenario::Classification; @@ -29,15 +29,12 @@ enum Command { Validate(SelectionArgs), /// Print one deterministic, fully expanded compiled scenario. Render(RenderArgs), - /// Boot one armed stack and the production Console for a browser test. - Serve(ServeArgs), + /// Boot one armed stack for a Playwright UI observe test. + Observe(ObserveArgs), } #[derive(Debug, Args)] -struct RunArgs { - #[command(flatten)] - selection: SelectionArgs, - +struct StackBinArgs { /// Path to the pinned iii engine binary. Falls back to $III_BIN. #[arg(long, env = "III_BIN")] engine_bin: Option, @@ -49,6 +46,15 @@ struct RunArgs { /// Real worker binaries as name=path. #[arg(long = "worker-bin", value_parser = parse_worker_bin)] worker_bins: Vec<(String, PathBuf)>, +} + +#[derive(Debug, Args)] +struct RunArgs { + #[command(flatten)] + selection: SelectionArgs, + + #[command(flatten)] + bins: StackBinArgs, /// Root directory for run artifacts. #[arg(long, default_value = "target/integration")] @@ -57,35 +63,17 @@ struct RunArgs { /// Keep heavyweight artifacts for passing scenarios. #[arg(long)] retain_success: bool, - - /// Run every selected scenario this many times and require an identical - /// stable result from each repetition. - #[arg(long, default_value_t = 1, value_parser = parse_repeat)] - repeat: u16, } #[derive(Debug, Args)] -struct ServeArgs { +struct ObserveArgs { #[command(flatten)] selection: SelectionArgs, - /// Path to the pinned iii engine binary. Falls back to $III_BIN. - #[arg(long, env = "III_BIN")] - engine_bin: Option, - - /// Path to the harness binary under test. - #[arg(long)] - harness_bin: Option, - - /// Path to the production Console binary under test. - #[arg(long)] - console_bin: Option, - - /// Real worker binaries as name=path. - #[arg(long = "worker-bin", value_parser = parse_worker_bin)] - worker_bins: Vec<(String, PathBuf)>, + #[command(flatten)] + bins: StackBinArgs, - /// File atomically published after the stack and Console are ready. + /// File atomically published after the stack is armed and ready. #[arg(long)] ready_file: PathBuf, @@ -117,16 +105,6 @@ fn parse_worker_bin(raw: &str) -> Result<(String, PathBuf), String> { Ok((name.to_string(), PathBuf::from(path))) } -fn parse_repeat(raw: &str) -> Result { - let repeat = raw - .parse::() - .map_err(|error| format!("invalid repeat count {raw:?}: {error}"))?; - if repeat == 0 { - return Err("repeat count must be at least 1".to_string()); - } - Ok(repeat) -} - fn main() { tracing_subscriber::fmt() .with_writer(std::io::stderr) @@ -144,7 +122,7 @@ async fn dispatch(cli: Cli) -> i32 { Command::Run(args) => return run(args).await, Command::Validate(args) => validate(args), Command::Render(args) => render(args), - Command::Serve(args) => return serve(args).await, + Command::Observe(args) => return observe(args).await, }; match result { Ok(message) => { @@ -175,17 +153,12 @@ async fn run(args: RunArgs) -> i32 { .any(|fixture| fixture.driver != ScenarioDriver::Direct) { eprintln!( - "runner_error: scenario {:?} is driven by the Console; use `serve`", + "runner_error: scenario {:?} is driven by Observe; use `observe`", args.selection.scenario ); return 3; } - let bins = match resolve_stack_bins( - args.engine_bin.as_deref(), - args.harness_bin.as_deref(), - None, - &args.worker_bins, - ) { + let bins = match resolve_stack_bins(&args.bins) { Ok(bins) => bins, Err(error) => { eprintln!("runner_error: {error:#}"); @@ -203,59 +176,35 @@ async fn run(args: RunArgs) -> i32 { let mut exit_code = 0; for fixture in &fixtures { let scenario_id = fixture.scenario.id.clone(); - let mut stable_result = None; - for repetition in 1..=args.repeat { - tracing::info!( - scenario = %scenario_id, - repetition, - repeat = args.repeat, - "running" - ); - let outcome = run_scenario(&bins, fixture, &artifacts_dir, args.retain_success).await; - let classification = outcome.result.classification; - let repetition_label = if args.repeat > 1 { - format!(" [{repetition}/{}]", args.repeat) - } else { - String::new() - }; - println!( - "{scenario_id}{repetition_label}: {}{} — run {} ({} ms), artifacts: {}", - classification_str(classification), - match &outcome.result.failure { - Some(failure) => format!(" — {failure}"), - None => String::new(), - }, - outcome.run_id, - outcome.duration_ms, - outcome.run_root.display(), - ); - exit_code = exit_code.max(classification.exit_code()); - - match &stable_result { - None => stable_result = Some(outcome.result), - Some(expected) if expected == &outcome.result => {} - Some(_) => { - eprintln!( - "runner_error: {scenario_id} produced a different stable result on repetition {repetition}" - ); - exit_code = exit_code.max(3); - } - } - } + tracing::info!(scenario = %scenario_id, "running"); + let outcome = run_scenario(&bins, fixture, &artifacts_dir, args.retain_success).await; + let classification = outcome.result.classification; + println!( + "{scenario_id}: {}{} — run {} ({} ms), artifacts: {}", + classification_str(classification), + match &outcome.result.failure { + Some(failure) => format!(" — {failure}"), + None => String::new(), + }, + outcome.run_id, + outcome.duration_ms, + outcome.run_root.display(), + ); + exit_code = exit_code.max(classification.exit_code()); } exit_code } -async fn serve(args: ServeArgs) -> i32 { +async fn observe(args: ObserveArgs) -> i32 { if args.selection.scenario == "all" { - eprintln!("runner_error: serve requires one scenario id or slug"); + eprintln!("runner_error: observe requires one scenario id or slug"); return 3; } let fixture = match load_fixtures(&args.selection, true).and_then(|fixtures| { fixtures .into_iter() .next() - .context("serve selector returned no scenario") + .context("observe selector returned no scenario") }) { Ok(fixture) => fixture, Err(error) => { @@ -263,16 +212,7 @@ async fn serve(args: ServeArgs) -> i32 { return 3; } }; - let Some(console_bin) = args.console_bin.as_deref() else { - eprintln!("runner_error: --console-bin is required"); - return 3; - }; - let bins = match resolve_stack_bins( - args.engine_bin.as_deref(), - args.harness_bin.as_deref(), - Some(console_bin), - &args.worker_bins, - ) { + let bins = match resolve_stack_bins(&args.bins) { Ok(bins) => bins, Err(error) => { eprintln!("runner_error: {error:#}"); @@ -287,7 +227,7 @@ async fn serve(args: ServeArgs) -> i32 { } }; - let outcome = serve_scenario(&bins, &fixture, &artifacts_dir, &args.ready_file).await; + let outcome = observe_scenario(&bins, &fixture, &artifacts_dir, &args.ready_file).await; println!( "{}: {} — run {} ({} ms), artifacts: {}", fixture.scenario.id, @@ -344,22 +284,20 @@ fn prepare_artifacts_dir(dir: &Path) -> anyhow::Result { .with_context(|| format!("resolving {}", dir.display())) } -fn resolve_stack_bins( - engine: Option<&Path>, - harness: Option<&Path>, - console: Option<&Path>, - worker_bins: &[(String, PathBuf)], -) -> anyhow::Result { - let engine = engine +fn resolve_stack_bins(args: &StackBinArgs) -> anyhow::Result { + let engine = args + .engine_bin + .as_deref() .ok_or_else(|| anyhow::anyhow!("--engine-bin (or III_BIN) is required; see engine.lock"))?; - let harness = harness.ok_or_else(|| anyhow::anyhow!("--harness-bin is required"))?; + let harness = args + .harness_bin + .as_deref() + .ok_or_else(|| anyhow::anyhow!("--harness-bin is required"))?; let bins = StackBins { engine: absolute_binary("engine", engine)?, harness: absolute_binary("harness", harness)?, - console: console - .map(|path| absolute_binary("console", path)) - .transpose()?, - workers: worker_bins + workers: args + .worker_bins .iter() .map(|(name, path)| Ok((name.clone(), absolute_binary(name, path)?))) .collect::>>()?, @@ -378,16 +316,3 @@ fn absolute_binary(name: &str, path: &Path) -> anyhow::Result { path.canonicalize() .with_context(|| format!("resolving {name} binary {}", path.display())) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn repeat_count_must_be_positive() { - assert_eq!(parse_repeat("1").unwrap(), 1); - assert_eq!(parse_repeat("2").unwrap(), 2); - assert!(parse_repeat("0").is_err()); - assert!(parse_repeat("many").is_err()); - } -} diff --git a/harness/evals/integration/src/matcher.rs b/harness/evals/integration/src/matcher.rs index 6633a98be..d688a822e 100644 --- a/harness/evals/integration/src/matcher.rs +++ b/harness/evals/integration/src/matcher.rs @@ -138,24 +138,7 @@ fn normalized_pair( fn apply_normalizer(doc: &mut Value, normalizer: &JsonNormalizerV1) -> anyhow::Result<()> { validate_pointer(&normalizer.pointer)?; match normalizer.operation { - NormalizerOperation::Replace => { - let replacement = normalizer - .replacement - .clone() - .ok_or_else(|| anyhow::anyhow!("replace normalizer requires `replacement`"))?; - if normalizer.pointer.is_empty() { - *doc = replacement; - return Ok(()); - } - if let Some(target) = doc.pointer_mut(&normalizer.pointer) { - *target = replacement; - } - Ok(()) - } NormalizerOperation::Delete => { - if normalizer.replacement.is_some() { - anyhow::bail!("delete normalizer forbids `replacement`"); - } if normalizer.pointer.is_empty() { anyhow::bail!("delete normalizer cannot target the document root"); } @@ -207,12 +190,6 @@ fn split_pointer(pointer: &str) -> (&str, String) { (&pointer[..idx], token) } -/// Public subset check for non-matcher consumers (readiness compares each -/// seeded configuration key against the worker's stored resolved config). -pub fn subset_of(expected: &Value, actual: &Value) -> Option { - subset_with_array_policy(expected, actual, ArrayPolicy::Prefix) -} - /// Array semantics for structural subset comparisons. Objects are always /// recursive subsets; callers must choose whether arrays may have an /// unmatched suffix or must match in full. @@ -361,7 +338,6 @@ mod tests { let normalize = vec![JsonNormalizerV1 { pointer: "/0/timestamp".into(), operation: NormalizerOperation::Delete, - replacement: None, }]; let m = exact( json!([{ "role": "user", "content": [{ "type": "text", "text": "hi" }] }]), @@ -411,7 +387,6 @@ mod tests { &JsonNormalizerV1 { pointer: "/missing/deep".into(), operation: NormalizerOperation::Delete, - replacement: None, }, ) .unwrap(); @@ -421,7 +396,6 @@ mod tests { &JsonNormalizerV1 { pointer: String::new(), operation: NormalizerOperation::Delete, - replacement: None, }, ) .is_err()); diff --git a/harness/evals/integration/src/process.rs b/harness/evals/integration/src/process.rs index 36e251a59..0282f3153 100644 --- a/harness/evals/integration/src/process.rs +++ b/harness/evals/integration/src/process.rs @@ -8,11 +8,9 @@ mod child; mod spec; mod supervisor; -pub use child::SupervisedChild; -pub use spec::ProcessSpec; -pub use supervisor::{ - EarlyExit, ProcessSupervisor, TeardownIssue, TeardownReport, DEFAULT_TEARDOWN_BUDGET, -}; +pub(crate) use spec::ProcessSpec; +pub use supervisor::EarlyExit; +pub(crate) use supervisor::{ProcessSupervisor, TeardownReport, DEFAULT_TEARDOWN_BUDGET}; #[cfg(test)] mod tests; diff --git a/harness/evals/integration/src/process/spec.rs b/harness/evals/integration/src/process/spec.rs index 1b96780ef..4b7d3b498 100644 --- a/harness/evals/integration/src/process/spec.rs +++ b/harness/evals/integration/src/process/spec.rs @@ -53,20 +53,6 @@ impl ProcessSpec { self } - pub fn envs(mut self, values: I) -> Self - where - I: IntoIterator, - K: Into, - V: Into, - { - self.env.extend( - values - .into_iter() - .map(|(key, value)| (key.into(), value.into())), - ); - self - } - pub(super) fn spawn(self) -> anyhow::Result { create_parent(&self.stdout_log)?; create_parent(&self.stderr_log)?; diff --git a/harness/evals/integration/src/process/supervisor.rs b/harness/evals/integration/src/process/supervisor.rs index c40850580..37cb0af7d 100644 --- a/harness/evals/integration/src/process/supervisor.rs +++ b/harness/evals/integration/src/process/supervisor.rs @@ -51,10 +51,6 @@ impl ProcessSupervisor { } } - pub fn teardown_budget(&self) -> Duration { - self.teardown_budget - } - pub fn set_teardown_budget(&mut self, teardown_budget: Duration) { self.teardown_budget = teardown_budget; } diff --git a/harness/evals/integration/src/readiness.rs b/harness/evals/integration/src/readiness.rs deleted file mode 100644 index e0a8dc326..000000000 --- a/harness/evals/integration/src/readiness.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Schema-based readiness: never sleep-based. The probe -//! retries until every surface is present or the deadline passes, then -//! reports **every** missing surface by name (classification `setup_error`). - -mod catalog; -mod contracts; -mod probe; -mod spec; - -pub use catalog::{ - config_failure, has_function, has_registered_trigger, missing_functions, missing_trigger_types, - registered_trigger_failures, topic_failures, -}; -pub use contracts::ExpectedTriggerBinding; -pub use spec::ReadinessSpec; - -pub(crate) use catalog::registered_trigger_count; -pub(crate) use contracts::{ - contract_failures, controlled_contracts, router_contract, ExpectedFunctionContract, -}; -pub(crate) use probe::{ - probe, registered_trigger_snapshot, wait_for_contracts, wait_for_registered_triggers, -}; -pub(crate) use spec::ReadinessReport; diff --git a/harness/evals/integration/src/readiness/catalog.rs b/harness/evals/integration/src/readiness/catalog.rs deleted file mode 100644 index 4a1c53ee4..000000000 --- a/harness/evals/integration/src/readiness/catalog.rs +++ /dev/null @@ -1,176 +0,0 @@ -use std::collections::BTreeSet; - -use serde_json::Value; - -use super::{ExpectedTriggerBinding, ReadinessSpec}; - -/// Pure check: required function ids against a discovery listing. -pub fn missing_functions(spec: &ReadinessSpec, listed: &Value) -> Vec { - let ids = collect_ids(listed, &["function_id", "id"]); - spec.functions - .iter() - .filter(|required| !ids.contains(&required.function_id)) - .map(|required| format!("function {}", required.function_id)) - .collect() -} - -/// Structured discovery checks reused by Arm polling. These deliberately -/// inspect descriptor ids rather than searching a serialized JSON blob. -pub fn has_function(listed: &Value, function_id: &str) -> bool { - collect_ids(listed, &["function_id", "id"]).contains(function_id) -} - -pub fn has_registered_trigger(listed: &Value, function_id: &str) -> bool { - collect_ids(listed, &["function_id", "id"]).contains(function_id) -} - -pub fn registered_trigger_failures( - expected: &[ExpectedTriggerBinding], - listed: &Value, -) -> Vec { - let rows = listed - .get("registered_triggers") - .and_then(Value::as_array) - .or_else(|| listed.as_array()) - .map(Vec::as_slice) - .unwrap_or_default(); - let mut failures = Vec::new(); - - for binding in expected { - let for_function = rows - .iter() - .filter(|row| { - row.get("function_id").and_then(Value::as_str) == Some(binding.function_id.as_str()) - }) - .collect::>(); - let exact = for_function - .iter() - .filter(|row| { - row.get("trigger_type").and_then(Value::as_str) - == Some(binding.trigger_type.as_str()) - && row.get("config") == Some(&binding.config) - }) - .count(); - if exact != 1 { - failures.push(format!( - "trigger binding {} -> {} with config {}: expected exactly one, got {exact}", - binding.trigger_type, binding.function_id, binding.config - )); - } - - let expected_for_function = expected - .iter() - .filter(|candidate| candidate.function_id == binding.function_id) - .count(); - if for_function.len() != expected_for_function { - failures.push(format!( - "trigger binding target {}: expected {expected_for_function} total registration(s), got {}", - binding.function_id, - for_function.len() - )); - } - } - failures.sort(); - failures.dedup(); - failures -} - -pub fn registered_trigger_count(listed: &Value, expected: &ExpectedTriggerBinding) -> usize { - listed - .get("registered_triggers") - .and_then(Value::as_array) - .or_else(|| listed.as_array()) - .into_iter() - .flatten() - .filter(|row| { - row.get("function_id").and_then(Value::as_str) == Some(expected.function_id.as_str()) - && row.get("trigger_type").and_then(Value::as_str) - == Some(expected.trigger_type.as_str()) - && row.get("config") == Some(&expected.config) - }) - .count() -} - -/// Pure check: required trigger types against a trigger-type listing. -pub fn missing_trigger_types(spec: &ReadinessSpec, listed: &Value) -> Vec { - let ids = collect_ids(listed, &["trigger_type", "id", "name", "type"]); - spec.trigger_types - .iter() - .filter(|required| !ids.contains(*required)) - .map(|required| format!("trigger type {required}")) - .collect() -} - -/// Pure check: required queue topics (name + broker type) against -/// `engine::queue::list_topics` output. -pub fn topic_failures(spec: &ReadinessSpec, listed: &Value) -> Vec { - let topics: Vec<(String, String)> = listed - .as_array() - .map(|items| { - items - .iter() - .filter_map(|topic| { - Some(( - topic.get("name")?.as_str()?.to_string(), - topic - .get("broker_type") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(), - )) - }) - .collect() - }) - .unwrap_or_default(); - let mut failures = Vec::new(); - for (topic, broker) in &spec.queue_topics { - match topics.iter().find(|(name, _)| name == topic) { - None => failures.push(format!("queue topic {topic}")), - Some((_, actual)) if actual != broker => failures.push(format!( - "queue topic {topic} broker type: expected {broker}, got {actual}" - )), - Some(_) => {} - } - } - failures -} - -/// Pure check: one seeded configuration entry against a -/// `configuration::get` response. -pub fn config_failure(id: &str, expected: &Value, resp: &Value) -> Option { - match resp.get("value") { - Some(value) => crate::matcher::subset_of(expected, value) - .map(|detail| format!("configuration {id}: seed not authoritative: {detail}")), - None => Some(format!("configuration {id}: no value")), - } -} - -/// Collect id strings from a list response of unknown exact shape: an array -/// of descriptors (or `{functions: [...]}`/`{items: [...]}`), each carrying -/// the id under one of `keys`. -fn collect_ids(listed: &Value, keys: &[&str]) -> BTreeSet { - let items: Vec<&Value> = match listed { - Value::Array(items) => items.iter().collect(), - Value::Object(map) => map - .values() - .find_map(|value| value.as_array()) - .map(|items| items.iter().collect()) - .unwrap_or_default(), - _ => Vec::new(), - }; - items - .iter() - .filter_map(|item| { - if let Some(text) = item.as_str() { - return Some(text.to_string()); - } - keys.iter() - .find_map(|key| item.get(key).and_then(Value::as_str)) - .map(String::from) - }) - .collect() -} - -pub(super) fn listed_ids(listed: &Value) -> BTreeSet { - collect_ids(listed, &["function_id", "id"]) -} diff --git a/harness/evals/integration/src/readiness/contracts.rs b/harness/evals/integration/src/readiness/contracts.rs deleted file mode 100644 index 3c5735674..000000000 --- a/harness/evals/integration/src/readiness/contracts.rs +++ /dev/null @@ -1,396 +0,0 @@ -use serde_json::Value; - -use crate::types::recorder::{ - LifecycleAcceptedV1, LifecycleEventV1, RecorderConfigV1, RecorderTargetV1, -}; - -/// Validation-relevant function contract expected from the live engine -/// registry. A missing optional field means that field is intentionally not -/// part of this readiness assertion. -#[derive(Debug, Clone, PartialEq)] -pub struct ExpectedFunctionContract { - pub function_id: String, - pub description: Option, - pub request_schema: Option, - pub response_schema: Option, - pub schema_comparison: SchemaComparison, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ExpectedTriggerBinding { - pub trigger_type: String, - pub function_id: String, - pub config: Value, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SchemaComparison { - /// Producer-owned mirrors may differ only in JSON Schema annotations - /// that do not affect validation. - AnnotationInsensitive, - /// Authored target schemas are model-visible and must survive registry - /// registration byte-for-byte after canonical object-key ordering. - Exact, -} - -impl ExpectedFunctionContract { - pub fn id_only(function_id: impl Into) -> Self { - Self { - function_id: function_id.into(), - description: None, - request_schema: None, - response_schema: None, - schema_comparison: SchemaComparison::AnnotationInsensitive, - } - } - - pub fn from_golden(raw: &str) -> Self { - let value: Value = serde_json::from_str(raw).expect("checked-in function golden is JSON"); - Self { - function_id: required_string(&value, "function_id"), - description: value - .get("description") - .and_then(Value::as_str) - .map(String::from), - request_schema: value.get("request_schema").cloned(), - response_schema: value.get("response_schema").cloned(), - schema_comparison: SchemaComparison::AnnotationInsensitive, - } - } -} - -pub(crate) fn router_contract(function_id: &str) -> ExpectedFunctionContract { - router_contracts() - .into_iter() - .find(|contract| contract.function_id == function_id) - .unwrap_or_else(|| panic!("no scripted-router contract for {function_id}")) -} - -pub(crate) fn router_contracts() -> Vec { - [ - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../llm-router/tests/golden/schemas/router.chat.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../llm-router/tests/golden/schemas/router.abort.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../llm-router/tests/golden/schemas/router.models.list.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../llm-router/tests/golden/schemas/router.models.get.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../llm-router/tests/golden/schemas/router.models.supports.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../llm-router/tests/golden/schemas/router.system_prompt.get.json" - )), - ] - .into_iter() - .map(ExpectedFunctionContract::from_golden) - .collect() -} - -pub(crate) fn pre_harness_contracts() -> Vec { - let mut contracts = [ - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../session-manager/tests/golden/schemas/session.messages.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../context-manager/tests/golden/schemas/context.assemble.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../context-manager/tests/golden/schemas/context.count-tokens.json" - )), - ] - .into_iter() - .map(ExpectedFunctionContract::from_golden) - .collect::>(); - contracts.extend(router_contracts()); - contracts.push(lifecycle_contract()); - // Queue readiness is asserted semantically by calling this function and - // validating its topic response. There is no worker-owned golden for its - // engine builtin descriptor. - contracts.push(ExpectedFunctionContract::id_only( - "engine::queue::list_topics", - )); - contracts -} - -pub(crate) fn harness_contracts() -> Vec { - [ - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../harness/tests/golden/schemas/harness.send.json" - )), - include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../harness/tests/golden/schemas/harness.status.json" - )), - ] - .into_iter() - .map(ExpectedFunctionContract::from_golden) - .collect() -} - -pub(crate) fn controlled_contracts(config: &RecorderConfigV1) -> Vec { - std::iter::once(&config.target) - .chain(config.extra_functions.iter()) - .map(controlled_contract) - .collect() -} - -fn controlled_contract(target: &RecorderTargetV1) -> ExpectedFunctionContract { - ExpectedFunctionContract { - function_id: target.function_id.clone(), - description: Some(target.description.clone()), - request_schema: Some(Value::Object(target.request_schema.clone())), - response_schema: Some(crate::recorder::const_response_schema(&target.response)), - schema_comparison: SchemaComparison::Exact, - } -} - -fn lifecycle_contract() -> ExpectedFunctionContract { - ExpectedFunctionContract { - function_id: "integration-recorder::lifecycle".to_string(), - description: Some("Durable sink for harness lifecycle trigger deliveries.".to_string()), - request_schema: Some(schema_for::()), - response_schema: Some(schema_for::()), - schema_comparison: SchemaComparison::Exact, - } -} - -pub fn contract_failures(expected: &ExpectedFunctionContract, actual: &Value) -> Vec { - let descriptor = actual.get("function").unwrap_or(actual); - let mut failures = Vec::new(); - - let actual_id = descriptor - .get("function_id") - .or_else(|| descriptor.get("id")) - .and_then(Value::as_str); - if actual_id != Some(expected.function_id.as_str()) { - failures.push(format!( - "function {} id: expected {:?}, got {:?}", - expected.function_id, expected.function_id, actual_id - )); - } - - if let Some(description) = &expected.description { - let actual_description = descriptor.get("description").and_then(Value::as_str); - if actual_description != Some(description.as_str()) { - failures.push(format!( - "function {} description mismatch: expected {:?}, got {:?}", - expected.function_id, description, actual_description - )); - } - } - - compare_schema( - &mut failures, - expected, - descriptor, - "request", - "request_schema", - "request_format", - expected.request_schema.as_ref(), - ); - compare_schema( - &mut failures, - expected, - descriptor, - "response", - "response_schema", - "response_format", - expected.response_schema.as_ref(), - ); - failures -} - -fn compare_schema( - failures: &mut Vec, - expected_contract: &ExpectedFunctionContract, - descriptor: &Value, - side: &str, - primary_key: &str, - compatibility_key: &str, - expected: Option<&Value>, -) { - let Some(expected) = expected else { - return; - }; - let actual = descriptor - .get(primary_key) - .or_else(|| descriptor.get(compatibility_key)) - .filter(|schema| !schema.is_null()); - let Some(actual) = actual else { - failures.push(format!( - "function {} {side} schema missing", - expected_contract.function_id - )); - return; - }; - let (expected, actual) = match expected_contract.schema_comparison { - SchemaComparison::AnnotationInsensitive => { - (validation_schema(expected), validation_schema(actual)) - } - SchemaComparison::Exact => (expected.clone(), actual.clone()), - }; - if actual != expected { - failures.push(format!( - "function {} {side} schema mismatch: expected_sha256={}, actual_sha256={}", - expected_contract.function_id, - crate::canonical::sha256_of_canonical(&expected), - crate::canonical::sha256_of_canonical(&actual) - )); - } -} - -/// Strip only JSON Schema annotation keywords. Validation-affecting -/// keywords, including `default` (part of the callable contract in this -/// repository), remain in the comparison. -pub fn validation_schema(schema: &Value) -> Value { - const ANNOTATIONS: &[&str] = &[ - "$comment", - "$schema", - "deprecated", - "description", - "examples", - "readOnly", - "title", - "writeOnly", - ]; - match schema { - Value::Object(map) => Value::Object( - map.iter() - .filter(|(key, _)| !ANNOTATIONS.contains(&key.as_str())) - .map(|(key, value)| (key.clone(), validation_schema(value))) - .collect(), - ), - Value::Array(values) => Value::Array(values.iter().map(validation_schema).collect()), - value => value.clone(), - } -} - -fn schema_for() -> Value { - serde_json::to_value(schemars::schema_for!(T)).expect("JSON schema serializes") -} - -fn required_string(value: &Value, key: &str) -> String { - value - .get(key) - .and_then(Value::as_str) - .unwrap_or_else(|| panic!("checked-in function golden is missing {key}")) - .to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn annotation_only_schema_changes_are_ignored() { - let left = json!({ - "$schema": "draft", - "title": "Left", - "type": "object", - "properties": { "value": { "type": "string", "description": "left" } } - }); - let right = json!({ - "title": "Right", - "type": "object", - "properties": { "value": { "type": "string", "description": "right" } } - }); - assert_eq!(validation_schema(&left), validation_schema(&right)); - } - - #[test] - fn validation_keywords_still_mismatch() { - let expected = ExpectedFunctionContract { - function_id: "test::function".into(), - description: None, - request_schema: Some(json!({ "type": "string" })), - response_schema: None, - schema_comparison: SchemaComparison::AnnotationInsensitive, - }; - let failures = contract_failures( - &expected, - &json!({ - "function_id": "test::function", - "request_schema": { "type": "integer" } - }), - ); - assert_eq!(failures.len(), 1); - assert!(failures[0].contains("request schema mismatch")); - } - - #[test] - fn exact_target_contract_keeps_property_descriptions() { - let expected = ExpectedFunctionContract { - function_id: "run::target".into(), - description: Some("target".into()), - request_schema: Some(json!({ - "type": "object", - "properties": { - "value": { "type": "string", "description": "authoritative" } - } - })), - response_schema: Some(json!({ "const": { "ok": true } })), - schema_comparison: SchemaComparison::Exact, - }; - let failures = contract_failures( - &expected, - &json!({ - "function_id": "run::target", - "description": "target", - "request_schema": { - "type": "object", - "properties": { - "value": { "type": "string", "description": "changed" } - } - }, - "response_schema": { "const": { "ok": true } } - }), - ); - assert!(failures - .iter() - .any(|failure| failure.contains("request schema mismatch"))); - } - - #[test] - fn exact_target_contract_checks_description_and_response_schema() { - let expected = ExpectedFunctionContract { - function_id: "run::target".into(), - description: Some("authoritative target".into()), - request_schema: Some(json!({ "type": "object" })), - response_schema: Some(json!({ "const": { "ok": true } })), - schema_comparison: SchemaComparison::Exact, - }; - let failures = contract_failures( - &expected, - &json!({ - "function_id": "run::target", - "description": "changed target", - "request_schema": { "type": "object" }, - "response_schema": { "const": { "ok": false } } - }), - ); - assert_eq!(failures.len(), 2); - assert!(failures - .iter() - .any(|failure| failure.contains("description mismatch"))); - assert!(failures - .iter() - .any(|failure| failure.contains("response schema mismatch"))); - } -} diff --git a/harness/evals/integration/src/readiness/probe.rs b/harness/evals/integration/src/readiness/probe.rs deleted file mode 100644 index fe3a2be8f..000000000 --- a/harness/evals/integration/src/readiness/probe.rs +++ /dev/null @@ -1,201 +0,0 @@ -use std::time::Duration; - -use serde_json::{json, Value}; - -use crate::client::{Client, DEFAULT_CALL_TIMEOUT_MS}; -use crate::deadline::Deadline; - -use super::catalog::{ - config_failure, missing_functions, missing_trigger_types, registered_trigger_failures, - topic_failures, -}; -use super::{ - contract_failures, ExpectedFunctionContract, ExpectedTriggerBinding, ReadinessReport, - ReadinessSpec, -}; - -const READINESS_POLL_INTERVAL: Duration = Duration::from_millis(250); -const DISCOVERY_POLL_INTERVAL: Duration = Duration::from_millis(200); - -/// Probe until ready or deadline. Returns the last report on timeout. -pub async fn probe( - client: &Client, - spec: &ReadinessSpec, - deadline: Deadline, -) -> Result<(), ReadinessReport> { - loop { - let report = probe_once(client, spec, deadline).await; - if report.missing.is_empty() { - return Ok(()); - } - if deadline.is_expired() { - return Err(report); - } - tokio::time::sleep(READINESS_POLL_INTERVAL.min(deadline.remaining())).await; - } -} - -/// Wait until every controlled function is present with its exact live -/// descriptor contract. -pub(crate) async fn wait_for_contracts( - client: &Client, - contracts: &[ExpectedFunctionContract], - deadline: Deadline, -) -> Result<(), ReadinessReport> { - loop { - let missing = inspect_function_contracts(client, contracts, deadline).await; - if missing.is_empty() { - return Ok(()); - } - if deadline.is_expired() { - return Err(ReadinessReport { missing }); - } - tokio::time::sleep(DISCOVERY_POLL_INTERVAL.min(deadline.remaining())).await; - } -} - -/// Wait until every bound function id is present in registered-trigger -/// discovery. -pub(crate) async fn wait_for_registered_triggers( - client: &Client, - bindings: &[ExpectedTriggerBinding], - deadline: Deadline, -) -> Result<(), ReadinessReport> { - loop { - let failures = match registered_trigger_snapshot(client, deadline).await { - Ok(listed) => registered_trigger_failures(bindings, &listed), - Err(error) => vec![format!("registered trigger discovery unavailable: {error}")], - }; - if failures.is_empty() { - return Ok(()); - } - if deadline.is_expired() { - return Err(ReadinessReport { missing: failures }); - } - tokio::time::sleep(DISCOVERY_POLL_INTERVAL.min(deadline.remaining())).await; - } -} - -pub(crate) async fn registered_trigger_snapshot( - client: &Client, - deadline: Deadline, -) -> anyhow::Result { - client - .call_with_deadline( - "engine::registered-triggers::list", - json!({ "include_internal": true }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - .map_err(anyhow::Error::msg) -} - -async fn probe_once(client: &Client, spec: &ReadinessSpec, deadline: Deadline) -> ReadinessReport { - let mut missing = Vec::new(); - - // 1. Discovery responds, and every required function id is registered. - match client - .call_with_deadline( - "engine::functions::list", - json!({ "include_internal": true }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - { - Ok(listed) => { - missing.extend(missing_functions(spec, &listed)); - let available = super::catalog::listed_ids(&listed); - let contracts = spec - .functions - .iter() - .filter(|contract| available.contains(&contract.function_id)) - .cloned() - .collect::>(); - missing.extend(inspect_function_contracts(client, &contracts, deadline).await); - } - Err(error) => missing.push(format!("engine::functions::list unavailable: {error}")), - } - - // 2. Trigger types. - if !spec.trigger_types.is_empty() { - match client - .call_with_deadline( - "engine::triggers::list", - json!({ "include_internal": true }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - { - Ok(listed) => missing.extend(missing_trigger_types(spec, &listed)), - Err(error) => missing.push(format!("engine::triggers::list unavailable: {error}")), - } - } - - // 3. Queue topics with broker type. - if !spec.queue_topics.is_empty() { - match client - .call_with_deadline( - "engine::queue::list_topics", - json!({}), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - { - Ok(listed) => missing.extend(topic_failures(spec, &listed)), - Err(error) => { - missing.push(format!("engine::queue::list_topics unavailable: {error}")); - } - } - } - - // 4. Seeded configuration entries are authoritative. Workers store their - // RESOLVED config (seed merged with defaults — observed on first boot), - // so the check is: every seeded key is present with exactly the seeded - // value. Recorded as a spec correction to the original byte-compare. - for (id, expected) in &spec.config_entries { - match client - .call_with_deadline( - "configuration::get", - json!({ "id": id }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - { - Ok(response) => missing.extend(config_failure(id, expected, &response)), - Err(error) => missing.push(format!("configuration {id} unavailable: {error}")), - } - } - - ReadinessReport { missing } -} - -async fn inspect_function_contracts( - client: &Client, - contracts: &[ExpectedFunctionContract], - deadline: Deadline, -) -> Vec { - let mut failures = Vec::new(); - for contract in contracts { - match client - .call_with_deadline( - "engine::functions::info", - json!({ "function_id": contract.function_id }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - { - Ok(actual) => failures.extend(contract_failures(contract, &actual)), - Err(error) => failures.push(format!( - "function {} descriptor unavailable: {error}", - contract.function_id - )), - } - } - failures -} diff --git a/harness/evals/integration/src/readiness/spec.rs b/harness/evals/integration/src/readiness/spec.rs deleted file mode 100644 index 94136d709..000000000 --- a/harness/evals/integration/src/readiness/spec.rs +++ /dev/null @@ -1,53 +0,0 @@ -use serde_json::Value; - -use super::contracts::{harness_contracts, pre_harness_contracts}; -use super::ExpectedFunctionContract; - -#[derive(Debug, Clone)] -pub struct ReadinessSpec { - /// Exact live function contracts that must be registered. Internal - /// functions are visible because discovery passes - /// `include_internal: true`. - pub functions: Vec, - /// Trigger types that must be registered (e.g. `harness::turn-completed`). - pub trigger_types: Vec, - /// Queue topics that must exist as (name, expected broker type). - pub queue_topics: Vec<(String, String)>, - /// `configuration::get` id → authoritative seeded subset. Every seeded - /// value must match canonically; worker-installed defaults may add fields. - pub config_entries: Vec<(String, Value)>, -} - -impl ReadinessSpec { - /// The surface required before Arm — everything except the harness, - /// which is spawned after Arm (see `stack::WORKER_START_ORDER`). - pub fn pre_harness(config_entries: Vec<(String, Value)>) -> Self { - Self { - functions: pre_harness_contracts(), - trigger_types: Vec::new(), - queue_topics: Vec::new(), - config_entries, - } - } - - /// The harness surface, probed after `Stack::spawn_harness` and before - /// Send: public functions, lifecycle trigger types, the provisioned - /// `harness-turn` topic, and the harness's own seeded config entry. - pub fn harness_surface(config_entries: Vec<(String, Value)>) -> Self { - Self { - functions: harness_contracts(), - trigger_types: vec![ - "harness::turn-started".to_string(), - "harness::turn-completed".to_string(), - ], - queue_topics: vec![("harness-turn".to_string(), "builtin".to_string())], - config_entries, - } - } -} - -#[derive(Debug)] -pub struct ReadinessReport { - /// Empty when ready. Each entry names one missing/mismatched surface. - pub missing: Vec, -} diff --git a/harness/evals/integration/src/recorder.rs b/harness/evals/integration/src/recorder.rs index 98e64af1e..5a27f2c8f 100644 --- a/harness/evals/integration/src/recorder.rs +++ b/harness/evals/integration/src/recorder.rs @@ -10,7 +10,6 @@ mod service; mod store; -pub(crate) use service::const_response_schema; pub use service::Recorder; #[cfg(test)] diff --git a/harness/evals/integration/src/recorder/service.rs b/harness/evals/integration/src/recorder/service.rs index 702f8968a..c0ff84073 100644 --- a/harness/evals/integration/src/recorder/service.rs +++ b/harness/evals/integration/src/recorder/service.rs @@ -165,9 +165,9 @@ impl Recorder { self.store.reset(run_id) } - /// Return an ordered in-process snapshot after the optional sequence. - pub fn snapshot(&self, after_sequence: Option) -> anyhow::Result> { - self.store.snapshot(after_sequence) + /// Return the ordered in-process snapshot. + pub fn snapshot(&self) -> anyhow::Result> { + self.store.snapshot() } /// Wait for the lifecycle trigger delivery without repeatedly calling a @@ -181,7 +181,7 @@ impl Recorder { loop { let notified = self.event_notify.notified(); if let Some(event) = self - .snapshot(None)? + .snapshot()? .into_iter() .find(|event| event.kind == RecorderEventKind::Lifecycle) { @@ -287,7 +287,7 @@ fn register_controlled_function( ); } -pub(crate) fn const_response_schema(response: &Value) -> Value { +fn const_response_schema(response: &Value) -> Value { json!({ "$schema": "http://json-schema.org/draft-07/schema#", "const": response diff --git a/harness/evals/integration/src/recorder/store.rs b/harness/evals/integration/src/recorder/store.rs index b40d87b02..cfacaf9d1 100644 --- a/harness/evals/integration/src/recorder/store.rs +++ b/harness/evals/integration/src/recorder/store.rs @@ -110,18 +110,9 @@ impl EventStore { Ok(sequence) } - pub(super) fn snapshot( - &self, - after_sequence: Option, - ) -> anyhow::Result> { - let after_sequence = after_sequence.unwrap_or(0); + pub(super) fn snapshot(&self) -> anyhow::Result> { let state = self.lock()?; - Ok(state - .events - .iter() - .filter(|event| event.sequence > after_sequence) - .cloned() - .collect()) + Ok(state.events.clone()) } fn lock(&self) -> anyhow::Result> { diff --git a/harness/evals/integration/src/recorder/tests.rs b/harness/evals/integration/src/recorder/tests.rs index 8f6ad87f6..d6a5e7a90 100644 --- a/harness/evals/integration/src/recorder/tests.rs +++ b/harness/evals/integration/src/recorder/tests.rs @@ -34,14 +34,10 @@ fn event_store_persists_ordered_events_and_resets_durably() { 2 ); - let snapshot = store.snapshot(None).expect("snapshot"); + let snapshot = store.snapshot().expect("snapshot"); assert_eq!(snapshot.len(), 2); assert_eq!(snapshot[0].sequence, 1); assert_eq!(snapshot[1].sequence, 2); - assert_eq!( - store.snapshot(Some(1)).expect("filtered snapshot"), - vec![snapshot[1].clone()] - ); let persisted: Vec = std::fs::read_to_string(&log_path) .expect("read log") @@ -57,7 +53,7 @@ fn event_store_persists_ordered_events_and_resets_durably() { ); assert_eq!(store.reset("run-2").expect("second reset"), 1); - assert!(store.snapshot(None).expect("empty snapshot").is_empty()); + assert!(store.snapshot().expect("empty snapshot").is_empty()); assert_eq!(std::fs::read(&log_path).expect("read reset log"), b""); assert_eq!( store @@ -81,7 +77,7 @@ fn failed_open_is_returned_without_acknowledging_the_event() { format!("{error:#}").contains("open recorder log for append"), "{error:#}" ); - assert!(store.snapshot(None).expect("snapshot").is_empty()); + assert!(store.snapshot().expect("snapshot").is_empty()); std::fs::create_dir(&parent).expect("create log parent"); assert_eq!( @@ -105,7 +101,7 @@ fn events_are_rejected_until_the_store_is_configured() { format!("{error:#}").contains("event store is not configured"), "{error:#}" ); - assert!(store.snapshot(None).expect("snapshot").is_empty()); + assert!(store.snapshot().expect("snapshot").is_empty()); } #[test] @@ -189,7 +185,7 @@ fn write_errors_are_returned_without_acknowledging_the_event() { format!("{error:#}").contains("write recorder log"), "{error:#}" ); - assert!(store.snapshot(None).expect("snapshot").is_empty()); + assert!(store.snapshot().expect("snapshot").is_empty()); } #[cfg(target_os = "linux")] @@ -205,5 +201,5 @@ fn fsync_errors_are_returned_without_acknowledging_the_event() { format!("{error:#}").contains("fsync recorder log"), "{error:#}" ); - assert!(store.snapshot(None).expect("snapshot").is_empty()); + assert!(store.snapshot().expect("snapshot").is_empty()); } diff --git a/harness/evals/integration/src/scenario.rs b/harness/evals/integration/src/scenario.rs index b43709a4c..53b6c6aa9 100644 --- a/harness/evals/integration/src/scenario.rs +++ b/harness/evals/integration/src/scenario.rs @@ -1,20 +1,22 @@ //! Scenario execution lifecycle: -//! Allocate → Boot → Probe → Arm → Send → Fault/Release → Await → +//! Allocate → Boot → Arm → Send → Fault/Release → Await → //! Collect → Grade → Teardown → Report. +//! (Observe inserts Probe/wait-start between Arm and Send, then waits for +//! observer shutdown after Await before Collect.) //! //! Every phase returns [`crate::runtime::RunError`]. Classification is derived //! once after process state has been inspected. pub mod floor; +mod observe; mod phases; mod report; mod runner; -mod serve; mod state; +pub use observe::{observe_scenario, ObserveOutcome, ObserveReadyV1, ObserveResultV1}; pub use runner::{run_scenario, RunOutcome}; -pub use serve::{serve_scenario, ServeOutcome, ServeReadyV1, ServeResultV1}; #[cfg(test)] mod tests; diff --git a/harness/evals/integration/src/scenario/floor.rs b/harness/evals/integration/src/scenario/floor.rs index 56ad93680..39d610abf 100644 --- a/harness/evals/integration/src/scenario/floor.rs +++ b/harness/evals/integration/src/scenario/floor.rs @@ -159,8 +159,8 @@ fn generations_failure(run: &RunEvidence) -> Option { }) } -/// Send accepted with clean flags. Skipped in serve mode, where the Console -/// (not the runner) submits the send and there is no response to inspect. +/// Send accepted with clean flags. Skipped only when `send_response` is +/// absent (should not happen for Direct or Observe after a successful Send). fn send_flags_failure(run: &RunEvidence) -> Option { let response = run.send_response.as_ref()?; // Absent optional flags normalize to false. @@ -243,7 +243,7 @@ mod tests { } #[test] - fn send_flags_normalize_absent_to_false_and_serve_skips_them() { + fn send_flags_normalize_absent_to_false_and_skip_without_response() { let mut evidence = clean_evidence(); evidence.send_response = Some(json!({ "accepted": true })); assert_eq!(floor_failure(&evidence), None); @@ -251,7 +251,7 @@ mod tests { evidence.send_response = Some(json!({ "accepted": true, "queued": true })); assert!(floor_failure(&evidence).unwrap().contains("queued: true")); - // Serve mode has no direct send response; the flags check is skipped. + // No send response yet (or collect failed early); the flags check is skipped. evidence.send_response = None; assert_eq!(floor_failure(&evidence), None); } diff --git a/harness/evals/integration/src/scenario/observe.rs b/harness/evals/integration/src/scenario/observe.rs new file mode 100644 index 000000000..796dde3df --- /dev/null +++ b/harness/evals/integration/src/scenario/observe.rs @@ -0,0 +1,391 @@ +//! Observe driver: armed stack for Playwright UI tests. +//! +//! The integration owns stimulus (`harness::send` after a start signal). +//! Playwright owns the Console process and DOM assertions. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::artifacts::write_json; +use crate::client::DEFAULT_CALL_TIMEOUT_MS; +use crate::deadline::Deadline; +use crate::fixtures::ScenarioFixture; +use crate::runtime::{RunError, RunErrorKind, RunPhase}; +use crate::scenarios::ScenarioDriver; +use crate::stack::{Stack, StackBins}; +use crate::types::scenario::{Classification, CompiledSendV1}; +use crate::types::script::SchemaVersion1; + +use super::runner::{BootedRun, ExpandedRun, ScenarioRunner}; +use super::state::PreparedRun; + +const START_POLL_INTERVAL: Duration = Duration::from_millis(100); +const PROCESS_POLL_INTERVAL: Duration = Duration::from_millis(100); + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObserveReadyV1 { + pub schema_version: SchemaVersion1, + pub run_id: String, + pub scenario_id: String, + pub scenario_slug: String, + pub driver: ScenarioDriver, + pub run_root: PathBuf, + pub result_path: PathBuf, + pub engine_url: String, + pub session: ObserveSessionV1, + pub model: ObserveModelV1, + pub message: String, + pub functions: BTreeMap, + pub send: CompiledSendV1, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObserveSessionV1 { + pub id: String, + pub title: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObserveModelV1 { + pub id: String, + pub provider: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObserveStartV1 { + pub schema_version: SchemaVersion1, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObserveResultV1 { + pub schema_version: SchemaVersion1, + pub scenario_id: String, + pub classification: Classification, + /// First floor or verify failure, scrubbed of run-scoped ids. + pub failure: Option, + /// Raw serialized [`crate::evidence_data::RunEvidence`] (JSON null when + /// the run failed before collection). Ids are real, so Playwright can + /// check evidence against the ready manifest. + pub evidence: serde_json::Value, + pub artifacts: Vec, +} + +pub struct ObserveOutcome { + pub result: ObserveResultV1, + pub run_id: String, + pub run_root: PathBuf, + pub duration_ms: u64, +} + +pub async fn observe_scenario( + bins: &StackBins, + fixture: &ScenarioFixture, + artifacts_dir: &Path, + ready_file: &Path, +) -> ObserveOutcome { + let started = std::time::Instant::now(); + let run_id = format!("iu{}", &uuid::Uuid::new_v4().simple().to_string()[..12]); + let run_root = artifacts_dir.join(&run_id); + let session_id = format!("s_{}", uuid::Uuid::new_v4().simple()); + + let mut runner = ScenarioRunner::new(bins, fixture, run_id.clone(), session_id); + + // The observe consumer contract is observe-ready.json / ready-file -> + // observe-result.json; the direct-run result.json/execution.json pair is + // not written here. + let mut classification = execute_observe(&mut runner, artifacts_dir, ready_file).await; + let duration_ms = started.elapsed().as_millis() as u64; + + let mut result = ObserveResultV1 { + schema_version: SchemaVersion1::V1, + scenario_id: fixture.scenario.id.clone(), + classification, + failure: runner.failure.clone(), + evidence: runner.evidence.clone(), + artifacts: runner + .sink + .as_ref() + .map(|sink| sink.paths().to_vec()) + .unwrap_or_default(), + }; + if let Err(error) = write_json(&run_root, &run_root.join("observe-result.json"), &result) { + tracing::error!(target: "harness_integration::scenario", "observe result failed: {error:#}"); + classification = classification.combine(Classification::RunnerError); + result.classification = classification; + let _ = write_json(&run_root, &run_root.join("observe-result.json"), &result); + } + + if classification == Classification::Pass { + if let Some(sink) = &runner.sink { + sink.trim_passing_run(); + } + } + + ObserveOutcome { + result, + run_id, + run_root, + duration_ms, + } +} + +async fn execute_observe( + runner: &mut ScenarioRunner<'_>, + artifacts_dir: &Path, + ready_file: &Path, +) -> Classification { + let ExpandedRun { paths, expanded } = match runner.expand_for_run(artifacts_dir) { + Ok(expanded) => expanded, + Err(classification) => return classification, + }; + + let mut booted = match runner.boot_prepared(paths, expanded).await { + Ok(booted) => booted, + Err(classification) => return classification, + }; + + let outcome = async { + runner.arm_booted(&mut booted).await?; + run_observe_phases(runner, &mut booted, ready_file).await + } + .await; + runner + .finalize( + booted.stack, + booted.services, + booted.teardown_budget, + outcome, + ) + .await +} + +async fn run_observe_phases( + runner: &mut ScenarioRunner<'_>, + booted: &mut BootedRun, + ready_file: &Path, +) -> Result<(), RunError> { + let stack = &mut booted.stack; + let services = &booted.services; + let prepared = &booted.prepared; + + let session_title = format!("Console E2E {} {}", prepared.scenario.id, runner.run_id); + services + .client() + .call_with_deadline( + "session::ensure", + json!({ + "session_id": runner.session_id, + "title": session_title, + "metadata": { + "surface": "console", + "model": format!( + "{}::{}", + prepared.scenario.send.provider, + prepared.scenario.send.model + ), + "mode": "agent", + "title_manual": true, + "integration_run_id": runner.run_id + } + }), + prepared.setup_deadline, + DEFAULT_CALL_TIMEOUT_MS, + ) + .await + .map_err(|error| { + RunError::setup( + RunPhase::Arm, + "ensure console test session", + anyhow::anyhow!(error), + ) + })?; + + let ready = build_ready_manifest(runner, prepared, stack, &session_title); + runner.write_run_artifact("observe-ready.json", &ready, RunPhase::Report)?; + write_atomic_json(ready_file, &ready).map_err(|error| { + RunError::runner(RunPhase::Report, "publish observe ready manifest", error) + })?; + + let start_file = start_file_path(ready_file)?; + wait_for_start(&start_file, prepared.setup_deadline).await?; + + let mut active = runner.send(services, prepared).await?; + runner.fault(stack, services, prepared, &active).await?; + runner.release(services, prepared, &active).await?; + runner.r#await(services, &mut active).await?; + + let scenario_deadline = active.deadline; + wait_for_shutdown(stack, scenario_deadline).await?; + + runner.collect(services, prepared, &mut active).await?; + let evidence = runner.build_evidence(services, &active, Some(active.send_response.clone())); + runner.evidence = serde_json::to_value(&evidence).map_err(|error| { + RunError::runner(RunPhase::Grade, "serialize observe run evidence", error) + })?; + runner.verify_evidence(services, &evidence, active.timed_out) +} + +fn build_ready_manifest( + runner: &ScenarioRunner<'_>, + prepared: &PreparedRun, + stack: &Stack, + session_title: &str, +) -> ObserveReadyV1 { + let prefix = format!("{}::", runner.run_id); + let functions = std::iter::once(&prepared.scenario.recorder.target) + .chain(prepared.scenario.recorder.extra_functions.iter()) + .filter_map(|function| { + function + .function_id + .strip_prefix(&prefix) + .map(|alias| (alias.to_string(), function.function_id.clone())) + }) + .collect(); + let run_root = stack.paths.root.clone(); + ObserveReadyV1 { + schema_version: SchemaVersion1::V1, + run_id: runner.run_id.clone(), + scenario_id: prepared.scenario.id.clone(), + scenario_slug: runner.fixture.slug.clone(), + driver: runner.fixture.driver, + result_path: run_root.join("observe-result.json"), + run_root, + engine_url: stack.ws_url.clone(), + session: ObserveSessionV1 { + id: runner.session_id.clone(), + title: session_title.to_string(), + }, + model: ObserveModelV1 { + id: prepared.scenario.send.model.clone(), + provider: prepared.scenario.send.provider.clone(), + }, + message: prepared.scenario.send.message.clone(), + functions, + send: prepared.scenario.send.clone(), + } +} + +fn start_file_path(ready_file: &Path) -> Result { + let parent = ready_file.parent().ok_or_else(|| { + RunError::new( + RunPhase::Probe, + RunErrorKind::Runner, + "ready file path has no parent directory for start.json", + ) + })?; + Ok(parent.join("start.json")) +} + +async fn wait_for_start(start_file: &Path, deadline: Deadline) -> Result<(), RunError> { + deadline + .poll_until("observer start signal", START_POLL_INTERVAL, || { + let start_file = start_file.to_path_buf(); + async move { + match std::fs::read_to_string(&start_file) { + Ok(contents) => { + let parsed: ObserveStartV1 = serde_json::from_str(&contents) + .map_err(|error| anyhow::anyhow!("invalid start.json: {error}"))?; + let _ = parsed; + Ok(Some(())) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(anyhow::anyhow!( + "read start signal {}: {error}", + start_file.display() + )), + } + } + }) + .await + .map_err(|error| { + let kind = if deadline.is_expired() { + RunErrorKind::Timeout + } else { + RunErrorKind::Setup + }; + RunError::with_source( + RunPhase::Probe, + kind, + "observer did not publish start.json", + error, + ) + }) +} + +async fn wait_for_shutdown(stack: &mut Stack, deadline: Deadline) -> Result<(), RunError> { + let shutdown = shutdown_signal(); + tokio::pin!(shutdown); + let mut health = tokio::time::interval(PROCESS_POLL_INTERVAL); + health.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + result = &mut shutdown => { + result.map_err(|error| { + RunError::runner(RunPhase::Await, "wait for observe test shutdown", error) + })?; + return Ok(()); + } + _ = tokio::time::sleep_until(deadline.expires_at()) => { + return Err(RunError::new( + RunPhase::Await, + RunErrorKind::Timeout, + "observe test did not finish before the scenario deadline", + )); + } + _ = health.tick() => { + if let Some(exit) = stack.early_exit() { + return Err(RunError::new( + RunPhase::Await, + RunErrorKind::ProcessCrash, + format!( + "{} exited while the observe test was running: {}", + exit.name, exit.status + ), + )); + } + } + } + } +} + +async fn shutdown_signal() -> std::io::Result<()> { + #[cfg(unix)] + { + let mut sigterm = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result, + _ = sigterm.recv() => Ok(()), + } + } + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await + } +} + +fn write_atomic_json(path: &Path, value: &impl Serialize) -> anyhow::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + std::fs::create_dir_all(parent)?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow::anyhow!("ready file needs a UTF-8 filename"))?; + let temporary = parent.join(format!(".{file_name}.{}.tmp", std::process::id())); + let encoded = crate::canonical::canonical_json_pretty(&serde_json::to_value(value)?); + std::fs::write(&temporary, encoded)?; + std::fs::rename(&temporary, path)?; + Ok(()) +} diff --git a/harness/evals/integration/src/scenario/phases/arm.rs b/harness/evals/integration/src/scenario/phases/arm.rs new file mode 100644 index 000000000..4ee96b812 --- /dev/null +++ b/harness/evals/integration/src/scenario/phases/arm.rs @@ -0,0 +1,95 @@ +use crate::discovery; +use crate::runtime::{RunError, RunErrorKind, RunPhase}; +use crate::services::RunServices; +use crate::stack::Stack; + +use super::super::runner::ScenarioRunner; +use super::super::state::PreparedRun; + +impl ScenarioRunner<'_> { + pub(in crate::scenario) async fn arm( + &mut self, + stack: &mut Stack, + services: &RunServices, + prepared: &PreparedRun, + ) -> Result<(), RunError> { + let phase = RunPhase::Arm; + let recorder = services.recorder(); + let scenario = &prepared.scenario; + let deadline = prepared.setup_deadline; + + recorder + .configure(&self.run_id, &scenario.recorder) + .map_err(|error| RunError::setup(phase, "configure controlled recorder", error))?; + + recorder + .reset(&self.run_id) + .map_err(|error| RunError::setup(phase, "reset controlled recorder", error))?; + if !recorder + .snapshot() + .map_err(|error| { + RunError::setup(phase, "snapshot controlled recorder after reset", error) + })? + .is_empty() + { + return Err(RunError::new( + phase, + RunErrorKind::Setup, + "recorder snapshot is not empty after reset", + )); + } + + stack + .spawn_harness(self.bins) + .map_err(|error| RunError::setup(phase, "spawn harness under test", error))?; + + // Harness and workers register asynchronously; wait before bind/send. + discovery::wait_for_functions(services.client(), discovery::TURN_SURFACE, deadline) + .await + .map_err(|error| RunError::setup(phase, "wait for turn function surface", error))?; + discovery::wait_for_trigger_types( + services.client(), + &["harness::turn-started", "harness::turn-completed"], + deadline, + ) + .await + .map_err(|error| RunError::setup(phase, "wait for harness trigger types", error))?; + + recorder + .bind_lifecycle( + scenario.recorder.lifecycle.trigger_type.as_str(), + &self.session_id, + ) + .await + .map_err(|error| RunError::setup(phase, "bind lifecycle recorder", error))?; + for binding in &scenario.bindings { + recorder + .bind( + &binding.trigger_type, + &binding.function_id, + binding.config.clone(), + ) + .await + .map_err(|error| { + RunError::setup( + phase, + format!( + "bind trigger {} to {}", + binding.trigger_type, binding.function_id + ), + error, + ) + })?; + } + + self.sink_mut(phase)? + .write_scenario_text( + &scenario.id, + "expected-system-prompt.txt", + &prepared.expected_prompt, + ) + .map_err(|error| RunError::runner(phase, "write expected system prompt", error))?; + + Ok(()) + } +} diff --git a/harness/evals/integration/src/scenario/phases/completion.rs b/harness/evals/integration/src/scenario/phases/completion.rs index 17853c3e0..c1ccdd75a 100644 --- a/harness/evals/integration/src/scenario/phases/completion.rs +++ b/harness/evals/integration/src/scenario/phases/completion.rs @@ -1,19 +1,16 @@ -use std::sync::{Arc, Mutex}; use std::time::Duration; -use serde_json::{json, Value}; +use serde_json::json; use crate::client::DEFAULT_CALL_TIMEOUT_MS; use crate::deadline::Deadline; -use crate::runtime::{RunError, RunErrorKind, RunPhase}; +use crate::runtime::{RunError, RunPhase}; use crate::services::RunServices; -use crate::types::recorder::RecorderEventKind; use super::super::runner::ScenarioRunner; use super::super::state::ActiveTurn; -use super::{STATUS_POLL_INTERVAL, TARGET_POLL_INTERVAL}; -const LIFECYCLE_GRACE: Duration = Duration::from_secs(3); +const FINAL_STATUS_TIMEOUT: Duration = Duration::from_secs(10); impl ScenarioRunner<'_> { pub(in crate::scenario) async fn r#await( @@ -23,79 +20,14 @@ impl ScenarioRunner<'_> { ) -> Result<(), RunError> { let phase = RunPhase::Await; let deadline = active.deadline; - let last_status = Arc::new(Mutex::new(Value::Null)); - let last_status_error = Arc::new(Mutex::new(None::)); - let session_id = self.session_id.clone(); - let terminal = deadline - .poll_until("terminal harness status", STATUS_POLL_INTERVAL, || { - let last_status = Arc::clone(&last_status); - let last_status_error = Arc::clone(&last_status_error); - let session_id = session_id.clone(); - async move { - let response = services - .client() - .call_with_deadline( - "harness::status", - json!({ "session_id": session_id }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await; - let status = match response { - Ok(status) => { - *last_status_error.lock().map_err(|_| { - anyhow::anyhow!("last status error lock poisoned") - })? = None; - status - } - Err(error) => { - *last_status_error.lock().map_err(|_| { - anyhow::anyhow!("last status error lock poisoned") - })? = Some(error); - return Ok(None); - } - }; - *last_status - .lock() - .map_err(|_| anyhow::anyhow!("last status lock poisoned"))? = - status.clone(); - let terminal = matches!( - status.get("status").and_then(Value::as_str), - Some("completed") | Some("failed") | Some("cancelled") - ); - Ok(terminal.then_some(status)) - } - }) - .await; - - match terminal { - Ok(status) => active.final_status = status, + // Completion is event-driven: Arm bound harness::turn-completed to the + // recorder. Once it arrives, make one status call as the durable-state + // confirmation checked by the floor. + match services.recorder().wait_for_lifecycle(deadline).await { + Ok(_) => {} Err(error) if deadline.is_expired() => { - if let Some(status_error) = last_status_error - .lock() - .map_err(|_| { - RunError::new( - phase, - RunErrorKind::Runner, - "last status error lock poisoned", - ) - })? - .clone() - { - return Err(RunError::runner( - phase, - "harness::status remained unavailable at the scenario deadline", - anyhow::anyhow!(status_error), - )); - } active.timed_out = true; - active.final_status = last_status - .lock() - .map_err(|_| { - RunError::new(phase, RunErrorKind::Runner, "last status lock poisoned") - })? - .clone(); tracing::error!( target: "harness_integration::scenario", "await timed out: {error:#}" @@ -105,31 +37,29 @@ impl ScenarioRunner<'_> { Err(error) => { return Err(RunError::runner( phase, - "poll terminal harness status", + "wait for harness::turn-completed delivery", error, )); } } - let grace_expires = - (tokio::time::Instant::now() + LIFECYCLE_GRACE).min(deadline.expires_at()); - let grace = Deadline::at(grace_expires); - if !grace.is_expired() { - let lifecycle = grace - .poll_until("lifecycle delivery grace", TARGET_POLL_INTERVAL, || async { - let events = services.recorder().snapshot(None)?; - Ok(events - .iter() - .any(|event| event.kind == RecorderEventKind::Lifecycle) - .then_some(())) - }) - .await; - if let Err(error) = lifecycle { - if !grace.is_expired() { - return Err(RunError::runner(phase, "poll lifecycle delivery", error)); - } - } - } + let evidence_deadline = Deadline::after(FINAL_STATUS_TIMEOUT); + active.final_status = services + .client() + .call_with_deadline( + "harness::status", + json!({ "session_id": self.session_id }), + evidence_deadline, + DEFAULT_CALL_TIMEOUT_MS, + ) + .await + .map_err(|error| { + RunError::runner( + phase, + "confirm terminal harness status", + anyhow::anyhow!(error), + ) + })?; Ok(()) } } diff --git a/harness/evals/integration/src/scenario/phases/evidence.rs b/harness/evals/integration/src/scenario/phases/evidence.rs index f4d25a2aa..1e5ef0eb6 100644 --- a/harness/evals/integration/src/scenario/phases/evidence.rs +++ b/harness/evals/integration/src/scenario/phases/evidence.rs @@ -51,7 +51,7 @@ impl ScenarioRunner<'_> { active.recorder_events = services .recorder() - .snapshot(None) + .snapshot() .map_err(|error| RunError::runner(phase, "snapshot recorder evidence", error))?; let (target_calls, lifecycle_events): (Vec<_>, Vec<_>) = active .recorder_events @@ -96,7 +96,7 @@ impl ScenarioRunner<'_> { /// Enforce the runner-owned floor, then run the scenario's `verify` /// function. The first failure is recorded with run-scoped ids scrubbed - /// so repeated runs stay byte-stable. + /// so persisted results stay byte-stable. pub(in crate::scenario) fn verify_evidence( &mut self, services: &RunServices, diff --git a/harness/evals/integration/src/scenario/phases/execution.rs b/harness/evals/integration/src/scenario/phases/execution.rs index 873d54bee..6d9957dae 100644 --- a/harness/evals/integration/src/scenario/phases/execution.rs +++ b/harness/evals/integration/src/scenario/phases/execution.rs @@ -2,21 +2,22 @@ use std::time::Duration; use serde_json::{json, Value}; -use crate::client::DEFAULT_CALL_TIMEOUT_MS; +use crate::client::{Client, DEFAULT_CALL_TIMEOUT_MS}; use crate::deadline::Deadline; -use crate::readiness::{self, ReadinessSpec}; +use crate::discovery; use crate::runtime::{RunError, RunErrorKind, RunPhase}; use crate::services::RunServices; -use crate::stack::{expected_config_entries, expected_harness_config_entry, Stack}; +use crate::stack::Stack; use crate::types::recorder::RecorderEventKind; -use crate::types::scenario::FaultKind; +use crate::types::scenario::{CompiledScenarioV1, FaultKind}; use super::super::report::rpc_failure; use super::super::runner::ScenarioRunner; use super::super::state::{ActiveTurn, PreparedRun}; -use super::{STATUS_POLL_INTERVAL, TARGET_POLL_INTERVAL}; const SEND_TIMEOUT_MS: u64 = 30_000; +const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(250); +const TARGET_POLL_INTERVAL: Duration = Duration::from_millis(50); impl ScenarioRunner<'_> { pub(in crate::scenario) async fn send( @@ -84,7 +85,7 @@ impl ScenarioRunner<'_> { let FaultKind::EngineSigkill = fault.kind; let observed = deadline .poll_until("fault trigger", TARGET_POLL_INTERVAL, || async { - let events = services.recorder().snapshot(None)?; + let events = services.recorder().snapshot()?; let count = events .iter() .filter(|event| { @@ -147,14 +148,13 @@ impl ScenarioRunner<'_> { stack.respawn_engine().map_err(|error| { RunError::runner(phase, "respawn engine after fault injection", error) })?; - self.restore_after_engine_restart(stack, services, prepared, deadline) + self.restore_after_engine_restart(services, prepared, deadline) .await?; Ok(()) } async fn restore_after_engine_restart( &mut self, - stack: &Stack, services: &RunServices, prepared: &PreparedRun, deadline: Deadline, @@ -162,46 +162,31 @@ impl ScenarioRunner<'_> { let phase = RunPhase::Fault; let scenario = &prepared.scenario; - let pre_harness = ReadinessSpec::pre_harness(expected_config_entries(&stack.paths)); - if let Err(report) = readiness::probe(services.client(), &pre_harness, deadline).await { - return Err(self.readiness_failed( - &scenario.id, - phase, - RunErrorKind::Runner, - "post_restart_pre_harness", - "base contracts did not recover after engine restart", - &report.missing, - )); - } - - let controlled = readiness::controlled_contracts(&scenario.recorder); - if let Err(report) = - readiness::wait_for_contracts(services.client(), &controlled, deadline).await - { - return Err(self.readiness_failed( - &scenario.id, - phase, - RunErrorKind::Runner, - "post_restart_controlled", - "controlled contracts did not recover after engine restart", - &report.missing, - )); - } - - let harness = ReadinessSpec::harness_surface(expected_harness_config_entry(&stack.paths)); - if let Err(report) = readiness::probe(services.client(), &harness, deadline).await { - return Err(self.readiness_failed( - &scenario.id, + discovery::wait_for_functions(services.client(), discovery::TURN_SURFACE, deadline) + .await + .map_err(|error| { + RunError::runner( + phase, + "wait for turn function surface after engine restart", + error, + ) + })?; + discovery::wait_for_trigger_types( + services.client(), + &["harness::turn-started", "harness::turn-completed"], + deadline, + ) + .await + .map_err(|error| { + RunError::runner( phase, - RunErrorKind::Runner, - "post_restart_harness", - "harness contracts did not recover after engine restart", - &report.missing, - )); - } + "wait for harness trigger types after engine restart", + error, + ) + })?; - let expected_bindings = super::expected_trigger_bindings(scenario, &self.session_id); - let registered = readiness::registered_trigger_snapshot(services.client(), deadline) + let expected_bindings = expected_trigger_bindings(scenario, &self.session_id); + let registered = registered_trigger_snapshot(services.client(), deadline) .await .map_err(|error| { RunError::runner( @@ -212,7 +197,7 @@ impl ScenarioRunner<'_> { })?; for (index, expected) in expected_bindings.iter().enumerate() { - match readiness::registered_trigger_count(®istered, expected) { + match registered_trigger_count(®istered, expected) { 1 => {} 0 if index == 0 => services .recorder() @@ -261,20 +246,6 @@ impl ScenarioRunner<'_> { } } } - - if let Err(report) = - readiness::wait_for_registered_triggers(services.client(), &expected_bindings, deadline) - .await - { - return Err(self.readiness_failed( - &scenario.id, - phase, - RunErrorKind::Runner, - "post_restart_bindings", - "trigger bindings did not recover after engine restart", - &report.missing, - )); - } Ok(()) } @@ -372,3 +343,65 @@ impl ScenarioRunner<'_> { } } } + +#[derive(Debug, Clone, PartialEq)] +struct ExpectedTriggerBinding { + trigger_type: String, + function_id: String, + config: Value, +} + +fn expected_trigger_bindings( + scenario: &CompiledScenarioV1, + session_id: &str, +) -> Vec { + std::iter::once(ExpectedTriggerBinding { + trigger_type: scenario + .recorder + .lifecycle + .trigger_type + .as_str() + .to_string(), + function_id: "integration-recorder::lifecycle".to_string(), + config: json!({ "session_id": session_id }), + }) + .chain( + scenario + .bindings + .iter() + .map(|binding| ExpectedTriggerBinding { + trigger_type: binding.trigger_type.clone(), + function_id: binding.function_id.clone(), + config: binding.config.clone(), + }), + ) + .collect() +} + +async fn registered_trigger_snapshot(client: &Client, deadline: Deadline) -> anyhow::Result { + client + .call_with_deadline( + "engine::registered-triggers::list", + json!({ "include_internal": true }), + deadline, + DEFAULT_CALL_TIMEOUT_MS, + ) + .await + .map_err(anyhow::Error::msg) +} + +fn registered_trigger_count(listed: &Value, expected: &ExpectedTriggerBinding) -> usize { + listed + .get("registered_triggers") + .and_then(Value::as_array) + .or_else(|| listed.as_array()) + .into_iter() + .flatten() + .filter(|row| { + row.get("function_id").and_then(Value::as_str) == Some(expected.function_id.as_str()) + && row.get("trigger_type").and_then(Value::as_str) + == Some(expected.trigger_type.as_str()) + && row.get("config") == Some(&expected.config) + }) + .count() +} diff --git a/harness/evals/integration/src/scenario/phases/mod.rs b/harness/evals/integration/src/scenario/phases/mod.rs index a2da5839f..75b2a186d 100644 --- a/harness/evals/integration/src/scenario/phases/mod.rs +++ b/harness/evals/integration/src/scenario/phases/mod.rs @@ -1,68 +1,4 @@ -use std::time::Duration; - -use serde_json::json; - -use crate::readiness::ExpectedTriggerBinding; -use crate::runtime::{RunError, RunErrorKind, RunPhase}; -use crate::types::scenario::CompiledScenarioV1; - -use super::runner::ScenarioRunner; - +mod arm; mod completion; mod evidence; mod execution; -mod readiness; - -const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(250); -const TARGET_POLL_INTERVAL: Duration = Duration::from_millis(50); - -impl ScenarioRunner<'_> { - /// Persist the readiness gap as `readiness-failure.json` and produce the - /// phase error, preferring the artifact-write failure when both fail. - fn readiness_failed( - &mut self, - scenario_id: &str, - phase: RunPhase, - kind: RunErrorKind, - stage: &str, - message: &str, - missing: &T, - ) -> RunError { - match self.write_artifact( - scenario_id, - "readiness-failure.json", - &json!({ "phase": stage, "missing": missing }), - phase, - ) { - Err(artifact_error) => artifact_error, - Ok(()) => RunError::new(phase, kind, message), - } - } -} - -fn expected_trigger_bindings( - scenario: &CompiledScenarioV1, - session_id: &str, -) -> Vec { - std::iter::once(ExpectedTriggerBinding { - trigger_type: scenario - .recorder - .lifecycle - .trigger_type - .as_str() - .to_string(), - function_id: "integration-recorder::lifecycle".to_string(), - config: json!({ "session_id": session_id }), - }) - .chain( - scenario - .bindings - .iter() - .map(|binding| ExpectedTriggerBinding { - trigger_type: binding.trigger_type.clone(), - function_id: binding.function_id.clone(), - config: binding.config.clone(), - }), - ) - .collect() -} diff --git a/harness/evals/integration/src/scenario/phases/readiness.rs b/harness/evals/integration/src/scenario/phases/readiness.rs deleted file mode 100644 index e59ac4a7a..000000000 --- a/harness/evals/integration/src/scenario/phases/readiness.rs +++ /dev/null @@ -1,155 +0,0 @@ -use crate::readiness::{self, ReadinessSpec}; -use crate::runtime::{RunError, RunErrorKind, RunPhase}; -use crate::services::RunServices; -use crate::stack::{expected_config_entries, expected_harness_config_entry, Stack}; - -use super::super::runner::ScenarioRunner; -use super::super::state::PreparedRun; - -impl ScenarioRunner<'_> { - pub(in crate::scenario) async fn probe( - &mut self, - stack: &mut Stack, - services: &RunServices, - prepared: &PreparedRun, - ) -> Result<(), RunError> { - let spec = ReadinessSpec::pre_harness(expected_config_entries(&stack.paths)); - if let Err(report) = - readiness::probe(services.client(), &spec, prepared.readiness_deadline).await - { - return Err(self.readiness_failed( - &prepared.scenario.id, - RunPhase::Probe, - RunErrorKind::Setup, - "pre_harness", - "pre-harness readiness failed", - &report.missing, - )); - } - Ok(()) - } - - pub(in crate::scenario) async fn arm( - &mut self, - stack: &mut Stack, - services: &RunServices, - prepared: &PreparedRun, - ) -> Result<(), RunError> { - let phase = RunPhase::Arm; - let recorder = services.recorder(); - let scenario = &prepared.scenario; - - recorder - .configure(&self.run_id, &scenario.recorder) - .map_err(|error| RunError::setup(phase, "configure controlled recorder", error))?; - - recorder - .reset(&self.run_id) - .map_err(|error| RunError::setup(phase, "reset controlled recorder", error))?; - if !recorder - .snapshot(None) - .map_err(|error| { - RunError::setup(phase, "snapshot controlled recorder after reset", error) - })? - .is_empty() - { - return Err(RunError::new( - phase, - RunErrorKind::Setup, - "recorder snapshot is not empty after reset", - )); - } - - let controlled_contracts = readiness::controlled_contracts(&scenario.recorder); - let discovery_deadline = prepared.readiness_deadline; - if let Err(report) = readiness::wait_for_contracts( - services.client(), - &controlled_contracts, - discovery_deadline, - ) - .await - { - return Err(self.readiness_failed( - &scenario.id, - phase, - RunErrorKind::Setup, - "controlled_functions", - "controlled function contracts did not match live discovery", - &report.missing, - )); - } - - stack - .spawn_harness(self.bins) - .map_err(|error| RunError::setup(phase, "spawn harness under test", error))?; - - let harness_deadline = prepared.readiness_deadline; - let spec = ReadinessSpec::harness_surface(expected_harness_config_entry(&stack.paths)); - if let Err(report) = readiness::probe(services.client(), &spec, harness_deadline).await { - return Err(self.readiness_failed( - &scenario.id, - phase, - RunErrorKind::Setup, - "harness", - "harness readiness failed", - &report.missing, - )); - } - - recorder - .bind_lifecycle( - scenario.recorder.lifecycle.trigger_type.as_str(), - &self.session_id, - ) - .await - .map_err(|error| RunError::setup(phase, "bind lifecycle recorder", error))?; - for binding in &scenario.bindings { - recorder - .bind( - &binding.trigger_type, - &binding.function_id, - binding.config.clone(), - ) - .await - .map_err(|error| { - RunError::setup( - phase, - format!( - "bind trigger {} to {}", - binding.trigger_type, binding.function_id - ), - error, - ) - })?; - } - - let expected_bindings = super::expected_trigger_bindings(scenario, &self.session_id); - let binding_deadline = prepared.readiness_deadline; - if let Err(report) = readiness::wait_for_registered_triggers( - services.client(), - &expected_bindings, - binding_deadline, - ) - .await - { - return Err(self.readiness_failed( - &scenario.id, - phase, - RunErrorKind::Setup, - "trigger_bindings", - "trigger bindings did not match live discovery", - &report.missing, - )); - } - - self.sink_mut(phase)? - .write_scenario_text( - &scenario.id, - "expected-system-prompt.txt", - &prepared.expected_prompt, - ) - .map_err(|error| RunError::runner(phase, "write expected system prompt", error))?; - - Ok(()) - } -} diff --git a/harness/evals/integration/src/scenario/runner.rs b/harness/evals/integration/src/scenario/runner.rs index 3f15cc8eb..967ed91d5 100644 --- a/harness/evals/integration/src/scenario/runner.rs +++ b/harness/evals/integration/src/scenario/runner.rs @@ -10,7 +10,7 @@ use crate::fixtures::ScenarioFixture; use crate::process::TeardownReport; use crate::runtime::{RunError, RunPhase}; use crate::services::RunServices; -use crate::stack::{EarlyExit, RunPaths, Stack, StackBins}; +use crate::stack::{EarlyExit, RunLayout, Stack, StackBins}; use crate::types::scenario::{Classification, ExecutionReportV1, IntegrationResultV1}; use crate::types::script::SchemaVersion1; @@ -72,15 +72,7 @@ pub async fn run_scenario( let run_root = artifacts_dir.join(&run_id); let session_id = format!("s_{}", uuid::Uuid::new_v4().simple()); - let mut runner = ScenarioRunner { - bins, - fixture, - run_id: run_id.clone(), - session_id, - sink: None, - failure: None, - evidence: serde_json::Value::Null, - }; + let mut runner = ScenarioRunner::new(bins, fixture, run_id.clone(), session_id); let mut classification = runner.execute(artifacts_dir).await; let mut result = runner.result(classification); @@ -140,33 +132,80 @@ pub(super) struct ScenarioRunner<'a> { /// First floor or verify failure, scrubbed of run-scoped ids. pub(super) failure: Option, /// Raw serialized [`crate::evidence_data::RunEvidence`], published in - /// serve results for Playwright; JSON null until collected. + /// observe results for Playwright; JSON null until collected. pub(super) evidence: serde_json::Value, } -impl ScenarioRunner<'_> { - async fn execute(&mut self, artifacts_dir: &Path) -> Classification { - let paths = match RunPaths::allocate(artifacts_dir, &self.run_id) { +/// Allocate + expand result shared by Direct and Observe drivers. +pub(super) struct ExpandedRun { + pub paths: RunLayout, + pub expanded: ExpandedFixtureV1, +} + +/// Booted stack + services ready for scenario phases. +pub(super) struct BootedRun { + pub stack: Stack, + pub services: RunServices, + pub prepared: PreparedRun, + pub teardown_budget: Duration, +} + +impl<'a> ScenarioRunner<'a> { + pub(super) fn new( + bins: &'a StackBins, + fixture: &'a ScenarioFixture, + run_id: String, + session_id: String, + ) -> Self { + Self { + bins, + fixture, + run_id, + session_id, + sink: None, + failure: None, + evidence: serde_json::Value::Null, + } + } + + /// Allocate run layout, open the artifact sink, and expand the fixture. + pub(super) fn expand_for_run( + &mut self, + artifacts_dir: &Path, + ) -> Result { + let paths = match RunLayout::allocate(artifacts_dir, &self.run_id) { Ok(paths) => paths, Err(error) => { let error = RunError::runner(RunPhase::Allocate, "allocate run paths", error); - return self.finish(Err(error), None); + return Err(self.finish(Err(error), None)); } }; self.sink = Some(ArtifactSink::new(paths.root.clone())); + let expanded = + match expand_compiled_fixture(&self.fixture.compiled(), &self.run_id, &self.session_id) + { + Ok(expanded) => expanded, + Err(error) => { + let error = + RunError::runner(RunPhase::Allocate, "expand compiled fixture", error); + return Err(self.finish_without_stack(error)); + } + }; + Ok(ExpandedRun { paths, expanded }) + } + + /// Create scenario dirs, boot the stack, and start support services. + pub(super) async fn boot_prepared( + &mut self, + paths: RunLayout, + expanded: ExpandedFixtureV1, + ) -> Result { let ExpandedFixtureV1 { scenario, script, system_prompt: expected_prompt, - } = match expand_compiled_fixture(&self.fixture.compiled(), &self.run_id, &self.session_id) - { - Ok(expanded) => expanded, - Err(error) => { - let error = RunError::runner(RunPhase::Allocate, "expand compiled fixture", error); - return self.finish_without_stack(error); - } - }; + } = expanded; let teardown_budget = Duration::from_millis(scenario.deadlines.teardown_ms); let prepared = PreparedRun::new(scenario, expected_prompt); @@ -176,14 +215,14 @@ impl ScenarioRunner<'_> { "allocate scenario artifact directory", error, ); - return self.finish_without_stack(error); + return Err(self.finish_without_stack(error)); } let mut stack = match Stack::boot(self.bins, paths).await { Ok(stack) => stack, Err(failure) => { let error = RunError::setup(RunPhase::Boot, "boot isolated stack", failure.error); - return self.fail_before_services(error, failure.teardown, None); + return Err(self.fail_before_services(error, failure.teardown, None)); } }; stack.set_teardown_budget(teardown_budget); @@ -201,23 +240,55 @@ impl ScenarioRunner<'_> { RunError::setup(RunPhase::Boot, "start run-scoped support services", error); let early_exit = stack.early_exit(); let processes = stack.teardown().await; - return self.fail_before_services(error, processes, early_exit); + return Err(self.fail_before_services(error, processes, early_exit)); } }; - let outcome = self.run_phases(&mut stack, &services, &prepared).await; - self.finalize(stack, services, teardown_budget, outcome) + Ok(BootedRun { + stack, + services, + prepared, + teardown_budget, + }) + } + + /// Arm a booted stack (shared by Direct and Observe drivers). + pub(super) async fn arm_booted(&mut self, booted: &mut BootedRun) -> Result<(), RunError> { + self.arm(&mut booted.stack, &booted.services, &booted.prepared) .await } - async fn run_phases( + async fn execute(&mut self, artifacts_dir: &Path) -> Classification { + let ExpandedRun { paths, expanded } = match self.expand_for_run(artifacts_dir) { + Ok(expanded) => expanded, + Err(classification) => return classification, + }; + let mut booted = match self.boot_prepared(paths, expanded).await { + Ok(booted) => booted, + Err(classification) => return classification, + }; + + let outcome = async { + self.arm_booted(&mut booted).await?; + self.run_phases_after_arm(&mut booted.stack, &booted.services, &booted.prepared) + .await + } + .await; + self.finalize( + booted.stack, + booted.services, + booted.teardown_budget, + outcome, + ) + .await + } + + async fn run_phases_after_arm( &mut self, stack: &mut Stack, services: &RunServices, prepared: &PreparedRun, ) -> Result<(), RunError> { - self.probe(stack, services, prepared).await?; - self.arm(stack, services, prepared).await?; let mut active = self.send(services, prepared).await?; self.fault(stack, services, prepared, &active).await?; self.release(services, prepared, &active).await?; @@ -246,7 +317,7 @@ impl ScenarioRunner<'_> { combine_teardown(classification, teardown_complete, artifact) } - /// Shared teardown tail for run and serve. Inspects process state while + /// Shared teardown tail for run and observe. Inspects process state while /// the subject is still running; service shutdown is intentionally /// before process teardown, and a second inspection catches a child /// that exits during that boundary. diff --git a/harness/evals/integration/src/scenario/serve.rs b/harness/evals/integration/src/scenario/serve.rs deleted file mode 100644 index 6a2230c30..000000000 --- a/harness/evals/integration/src/scenario/serve.rs +++ /dev/null @@ -1,507 +0,0 @@ -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; - -use crate::artifacts::{write_json, ArtifactSink}; -use crate::client::DEFAULT_CALL_TIMEOUT_MS; -use crate::deadline::Deadline; -use crate::expand::{expand_compiled_fixture, ExpandedFixtureV1}; -use crate::fixtures::ScenarioFixture; -use crate::runtime::{RunError, RunErrorKind, RunPhase}; -use crate::scenarios::ScenarioDriver; -use crate::services::RunServices; -use crate::stack::{free_loopback_port, RunPaths, Stack, StackBins}; -use crate::types::scenario::{Classification, CompiledSendV1}; -use crate::types::script::SchemaVersion1; - -use super::runner::ScenarioRunner; -use super::state::{ActiveTurn, PreparedRun}; - -const READY_POLL_INTERVAL: Duration = Duration::from_millis(100); -const PROCESS_POLL_INTERVAL: Duration = Duration::from_millis(100); -const FINAL_STATUS_TIMEOUT: Duration = Duration::from_secs(10); - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ServeReadyV1 { - pub schema_version: SchemaVersion1, - pub run_id: String, - pub scenario_id: String, - pub scenario_slug: String, - pub driver: ScenarioDriver, - pub run_root: PathBuf, - pub result_path: PathBuf, - pub console_url: String, - pub engine_url: String, - pub session: ServeSessionV1, - pub model: ServeModelV1, - pub message: String, - pub functions: BTreeMap, - #[serde(skip_serializing_if = "Option::is_none")] - pub send: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ServeSessionV1 { - pub id: String, - pub title: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ServeModelV1 { - pub id: String, - pub provider: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ServeResultV1 { - pub schema_version: SchemaVersion1, - pub scenario_id: String, - pub classification: Classification, - /// First floor or verify failure, scrubbed of run-scoped ids. - pub failure: Option, - /// Raw serialized [`crate::evidence_data::RunEvidence`] (JSON null when - /// the run failed before collection). Ids are real, so Playwright can - /// check evidence against the ready manifest. - pub evidence: serde_json::Value, - pub artifacts: Vec, -} - -pub struct ServeOutcome { - pub result: ServeResultV1, - pub run_id: String, - pub run_root: PathBuf, - pub duration_ms: u64, -} - -pub async fn serve_scenario( - bins: &StackBins, - fixture: &ScenarioFixture, - artifacts_dir: &Path, - ready_file: &Path, -) -> ServeOutcome { - let started = std::time::Instant::now(); - let run_id = format!("iu{}", &uuid::Uuid::new_v4().simple().to_string()[..12]); - let run_root = artifacts_dir.join(&run_id); - let session_id = format!("s_{}", uuid::Uuid::new_v4().simple()); - - let mut runner = ScenarioRunner { - bins, - fixture, - run_id: run_id.clone(), - session_id, - sink: None, - failure: None, - evidence: Value::Null, - }; - - // The serve consumer contract is serve-ready.json -> serve-result.json; - // the direct-run result.json/execution.json pair is not written here. - let mut classification = execute_serve(&mut runner, artifacts_dir, ready_file).await; - let duration_ms = started.elapsed().as_millis() as u64; - - let mut result = ServeResultV1 { - schema_version: SchemaVersion1::V1, - scenario_id: fixture.scenario.id.clone(), - classification, - failure: runner.failure.clone(), - evidence: runner.evidence.clone(), - artifacts: runner - .sink - .as_ref() - .map(|sink| sink.paths().to_vec()) - .unwrap_or_default(), - }; - if let Err(error) = write_json(&run_root, &run_root.join("serve-result.json"), &result) { - tracing::error!(target: "harness_integration::scenario", "serve result failed: {error:#}"); - classification = classification.combine(Classification::RunnerError); - result.classification = classification; - let _ = write_json(&run_root, &run_root.join("serve-result.json"), &result); - } - - if classification == Classification::Pass { - if let Some(sink) = &runner.sink { - sink.trim_passing_run(); - } - } - - ServeOutcome { - result, - run_id, - run_root, - duration_ms, - } -} - -async fn execute_serve( - runner: &mut ScenarioRunner<'_>, - artifacts_dir: &Path, - ready_file: &Path, -) -> Classification { - let paths = match RunPaths::allocate(artifacts_dir, &runner.run_id) { - Ok(paths) => paths, - Err(error) => { - return RunError::runner(RunPhase::Allocate, "allocate console run paths", error) - .classification(); - } - }; - runner.sink = Some(ArtifactSink::new(paths.root.clone())); - - let ExpandedFixtureV1 { - scenario, - script, - system_prompt: expected_prompt, - } = match expand_compiled_fixture( - &runner.fixture.compiled(), - &runner.run_id, - &runner.session_id, - ) { - Ok(expanded) => expanded, - Err(error) => { - let error = RunError::runner(RunPhase::Allocate, "expand console fixture", error); - return runner.finish(Err(error), None); - } - }; - - if scenario.fault.is_some() || scenario.release.is_some() { - let error = RunError::new( - RunPhase::Allocate, - RunErrorKind::Setup, - "serve scenarios cannot inject faults or hold responses", - ); - return runner.finish(Err(error), None); - } - let teardown_budget = Duration::from_millis(scenario.deadlines.teardown_ms); - let prepared = PreparedRun::new(scenario, expected_prompt); - if let Err(error) = paths.scenario_dir(&prepared.scenario.id) { - let error = RunError::runner( - RunPhase::Allocate, - "allocate console scenario artifact directory", - error, - ); - return runner.finish(Err(error), None); - } - - let mut stack = match Stack::boot(runner.bins, paths).await { - Ok(stack) => stack, - Err(failure) => { - let error = - RunError::setup(RunPhase::Boot, "boot isolated console stack", failure.error); - return runner.fail_before_services(error, failure.teardown, None); - } - }; - stack.set_teardown_budget(teardown_budget); - - let services = match RunServices::start( - &stack.ws_url, - script, - stack.paths.root.join("recorder.log.jsonl"), - ) - .await - { - Ok(services) => services, - Err(error) => { - let error = RunError::setup(RunPhase::Boot, "start console support services", error); - let early_exit = stack.early_exit(); - let processes = stack.teardown().await; - return runner.fail_before_services(error, processes, early_exit); - } - }; - - let outcome = run_serve_phases(runner, &mut stack, &services, &prepared, ready_file).await; - runner - .finalize(stack, services, teardown_budget, outcome) - .await -} - -async fn run_serve_phases( - runner: &mut ScenarioRunner<'_>, - stack: &mut Stack, - services: &RunServices, - prepared: &PreparedRun, - ready_file: &Path, -) -> Result<(), RunError> { - runner.probe(stack, services, prepared).await?; - runner.arm(stack, services, prepared).await?; - - let session_title = format!("Console E2E {} {}", prepared.scenario.id, runner.run_id); - services - .client() - .call_with_deadline( - "session::ensure", - json!({ - "session_id": runner.session_id, - "title": session_title, - "metadata": { - "surface": "console", - "model": format!( - "{}::{}", - prepared.scenario.send.provider, - prepared.scenario.send.model - ), - "mode": "agent", - "title_manual": true, - "integration_run_id": runner.run_id - } - }), - prepared.readiness_deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - .map_err(|error| { - RunError::setup( - RunPhase::Arm, - "ensure console test session", - anyhow::anyhow!(error), - ) - })?; - - let http_port = free_loopback_port() - .map_err(|error| RunError::setup(RunPhase::Arm, "allocate console HTTP port", error))?; - stack - .spawn_console(runner.bins, http_port) - .map_err(|error| RunError::setup(RunPhase::Arm, "spawn production console", error))?; - wait_for_console( - services, - &stack.ws_url, - http_port, - prepared.readiness_deadline, - ) - .await?; - - let ready = build_ready_manifest(runner, prepared, stack, http_port, &session_title); - runner.write_run_artifact("serve-ready.json", &ready, RunPhase::Report)?; - write_atomic_json(ready_file, &ready).map_err(|error| { - RunError::runner(RunPhase::Report, "publish console ready manifest", error) - })?; - - let scenario_deadline = Deadline::after(Duration::from_millis( - prepared.scenario.deadlines.scenario_ms, - )); - wait_for_shutdown(stack, scenario_deadline).await?; - - // Completion is event-driven: Arm bound harness::turn-completed to the - // recorder. Once it arrives, make one status call as the durable-state - // confirmation checked by the floor. - let lifecycle = services - .recorder() - .wait_for_lifecycle(scenario_deadline) - .await - .map_err(|error| { - let kind = if scenario_deadline.is_expired() { - RunErrorKind::Timeout - } else { - RunErrorKind::Runner - }; - RunError::with_source( - RunPhase::Await, - kind, - "wait for harness::turn-completed delivery", - error, - ) - })?; - let turn_id = lifecycle - .payload - .get("turn_id") - .and_then(Value::as_str) - .map(String::from); - let evidence_deadline = Deadline::after(FINAL_STATUS_TIMEOUT); - let final_status = services - .client() - .call_with_deadline( - "harness::status", - json!({ "session_id": runner.session_id }), - evidence_deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - .map_err(|error| { - RunError::runner( - RunPhase::Collect, - "confirm terminal harness status", - anyhow::anyhow!(error), - ) - })?; - - let mut active = ActiveTurn::new(evidence_deadline, turn_id, Value::Null); - active.final_status = final_status; - runner.collect(services, prepared, &mut active).await?; - - // Shared floor + verify path with Rpc mode. The Console (not the runner) - // submitted the send, so there is no send response and the send-flags - // floor check is skipped. - let evidence = runner.build_evidence(services, &active, None); - runner.evidence = serde_json::to_value(&evidence).map_err(|error| { - RunError::runner(RunPhase::Grade, "serialize console run evidence", error) - })?; - runner.verify_evidence(services, &evidence, false) -} - -fn build_ready_manifest( - runner: &ScenarioRunner<'_>, - prepared: &PreparedRun, - stack: &Stack, - http_port: u16, - session_title: &str, -) -> ServeReadyV1 { - let prefix = format!("{}::", runner.run_id); - let functions = std::iter::once(&prepared.scenario.recorder.target) - .chain(prepared.scenario.recorder.extra_functions.iter()) - .filter_map(|function| { - function - .function_id - .strip_prefix(&prefix) - .map(|alias| (alias.to_string(), function.function_id.clone())) - }) - .collect(); - let run_root = stack.paths.root.clone(); - ServeReadyV1 { - schema_version: SchemaVersion1::V1, - run_id: runner.run_id.clone(), - scenario_id: prepared.scenario.id.clone(), - scenario_slug: runner.fixture.slug.clone(), - driver: runner.fixture.driver, - result_path: run_root.join("serve-result.json"), - run_root, - console_url: format!("http://127.0.0.1:{http_port}"), - engine_url: stack.ws_url.clone(), - session: ServeSessionV1 { - id: runner.session_id.clone(), - title: session_title.to_string(), - }, - model: ServeModelV1 { - id: prepared.scenario.send.model.clone(), - provider: prepared.scenario.send.provider.clone(), - }, - message: prepared.scenario.send.message.clone(), - functions, - send: (runner.fixture.driver == ScenarioDriver::Direct) - .then(|| prepared.scenario.send.clone()), - } -} - -async fn wait_for_console( - services: &RunServices, - engine_url: &str, - http_port: u16, - deadline: Deadline, -) -> Result<(), RunError> { - deadline - .poll_until("console readiness", READY_POLL_INTERVAL, || async { - let status = services - .client() - .call_with_deadline( - "console::status", - json!({}), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await; - let status_ready = status.as_ref().is_ok_and(|value| { - value.get("http_port").and_then(Value::as_u64) == Some(u64::from(http_port)) - && value.get("engine_url").and_then(Value::as_str) == Some(engine_url) - }); - if !status_ready { - return Ok(None); - } - Ok(http_root_available(http_port).await.then_some(())) - }) - .await - .map_err(|error| { - RunError::setup( - RunPhase::Probe, - "production console did not become ready", - error, - ) - }) -} - -async fn http_root_available(port: u16) -> bool { - let Ok(mut stream) = tokio::net::TcpStream::connect(("127.0.0.1", port)).await else { - return false; - }; - if stream - .write_all(b"GET / HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n") - .await - .is_err() - { - return false; - } - let mut response = [0_u8; 64]; - let Ok(read) = stream.read(&mut response).await else { - return false; - }; - let status = String::from_utf8_lossy(&response[..read]); - status.starts_with("HTTP/1.1 200") || status.starts_with("HTTP/1.0 200") -} - -async fn wait_for_shutdown(stack: &mut Stack, deadline: Deadline) -> Result<(), RunError> { - let shutdown = shutdown_signal(); - tokio::pin!(shutdown); - let mut health = tokio::time::interval(PROCESS_POLL_INTERVAL); - health.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - tokio::select! { - result = &mut shutdown => { - result.map_err(|error| RunError::runner(RunPhase::Await, "wait for console test shutdown", error))?; - return Ok(()); - } - _ = tokio::time::sleep_until(deadline.expires_at()) => { - return Err(RunError::new( - RunPhase::Await, - RunErrorKind::Timeout, - "console test did not finish before the scenario deadline", - )); - } - _ = health.tick() => { - if let Some(exit) = stack.early_exit() { - return Err(RunError::new( - RunPhase::Await, - RunErrorKind::ProcessCrash, - format!("{} exited while the console test was running: {}", exit.name, exit.status), - )); - } - } - } - } -} - -async fn shutdown_signal() -> std::io::Result<()> { - #[cfg(unix)] - { - let mut sigterm = - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; - tokio::select! { - result = tokio::signal::ctrl_c() => result, - _ = sigterm.recv() => Ok(()), - } - } - #[cfg(not(unix))] - { - tokio::signal::ctrl_c().await - } -} - -fn write_atomic_json(path: &Path, value: &impl Serialize) -> anyhow::Result<()> { - let parent = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - std::fs::create_dir_all(parent)?; - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| anyhow::anyhow!("ready file needs a UTF-8 filename"))?; - let temporary = parent.join(format!(".{file_name}.{}.tmp", std::process::id())); - let encoded = crate::canonical::canonical_json_pretty(&serde_json::to_value(value)?); - std::fs::write(&temporary, encoded)?; - std::fs::rename(&temporary, path)?; - Ok(()) -} diff --git a/harness/evals/integration/src/scenario/state.rs b/harness/evals/integration/src/scenario/state.rs index 07a3bd73a..e5596e212 100644 --- a/harness/evals/integration/src/scenario/state.rs +++ b/harness/evals/integration/src/scenario/state.rs @@ -8,18 +8,18 @@ use crate::types::scenario::CompiledScenarioV1; pub(super) struct PreparedRun { pub(super) scenario: CompiledScenarioV1, pub(super) expected_prompt: String, - pub(super) readiness_deadline: Deadline, + pub(super) setup_deadline: Deadline, } impl PreparedRun { pub(super) fn new(scenario: CompiledScenarioV1, expected_prompt: String) -> Self { - let readiness_deadline = Deadline::after(std::time::Duration::from_millis( + let setup_deadline = Deadline::after(std::time::Duration::from_millis( scenario.deadlines.readiness_ms, )); Self { scenario, expected_prompt, - readiness_deadline, + setup_deadline, } } } diff --git a/harness/evals/integration/src/scenarios/builder.rs b/harness/evals/integration/src/scenarios/builder.rs index 7d958af7f..e7b012b11 100644 --- a/harness/evals/integration/src/scenarios/builder.rs +++ b/harness/evals/integration/src/scenarios/builder.rs @@ -252,18 +252,6 @@ impl FunctionCallReply { self.usage = Some(input_output_usage(input, output)); self } - - pub fn match_overrides(self, overrides: GenerationMatchOverridesV1) -> ScenarioGenerationV1 { - with_overrides(self.into(), overrides) - } - - /// Match this reply against the durable outcome only, at a recovery - /// boundary (fault restart or hook release) where the engine may rebuild - /// the request differently. Shorthand for [`Self::match_overrides`] with - /// the recovery policy. - pub fn recovery_boundary(self) -> ScenarioGenerationV1 { - with_overrides(self.into(), recovery_overrides()) - } } impl From for ScenarioGenerationV1 { @@ -379,8 +367,6 @@ macro_rules! impl_into_generations { impl_into_generations! { (G1); (G1, G2); - (G1, G2, G3); - (G1, G2, G3, G4); } pub fn regex(pattern: &str) -> JsonMatcherV1 { diff --git a/harness/evals/integration/src/scenarios/console_streamed_text.rs b/harness/evals/integration/src/scenarios/console_streamed_text.rs index a9ab9c2bd..e618447e6 100644 --- a/harness/evals/integration/src/scenarios/console_streamed_text.rs +++ b/harness/evals/integration/src/scenarios/console_streamed_text.rs @@ -1,33 +1,22 @@ -//! UI-001 — the production Console sends through its agent-trigger policy. +//! UI-001 — integration starts a streamed turn; Playwright validates Console UI. use anyhow::ensure; -use serde_json::json; - -use crate::types::scenario::GenerationMatchOverridesV1; use super::builder::*; pub(super) fn scenario() -> Scenario { AuthoredScenario::new( "UI-001", - "A message sent from the Console streams to durable completion.", + "A harness-started streamed turn renders to durable completion in the Console.", ) .trigger(Harness::send("Return the console fixture phrase.")) .model((Reply::text("console fixture complete") .chunks(["console fixture ", "complete"]) - .usage(9, 3) - .match_overrides(GenerationMatchOverridesV1 { - // The Console supplies agent mode and its production function - // policy, so its composed prompt intentionally differs from - // the direct integration request's native-policy golden. - system_prompt: Some(regex("agent_trigger")), - tools: Some(subset(json!([{ "name": "agent_trigger" }]))), - ..Default::default() - }),)) + .usage(9, 3),)) // Content assertions for UI scenarios live in Playwright (`ui-send` checks // the rendered text and both message counts in the DOM); the floor (turn - // completion, script consumption) is runner-owned, so this only checks what - // the DOM cannot show. + // completion, script consumption, clean send) is runner-owned, so this + // only checks what the DOM cannot show. .verify(|run| { ensure!( !run.has_duplicate_messages(), diff --git a/harness/evals/integration/src/scenarios/mod.rs b/harness/evals/integration/src/scenarios/mod.rs index 3a1c3511e..c288aacc2 100644 --- a/harness/evals/integration/src/scenarios/mod.rs +++ b/harness/evals/integration/src/scenarios/mod.rs @@ -31,7 +31,7 @@ pub type VerifyFn = fn(&RunEvidence) -> anyhow::Result<()>; #[serde(rename_all = "snake_case")] pub enum ScenarioDriver { Direct, - Console, + Observe, } /// One authored scenario, its verify function, and the stable slug used by @@ -48,7 +48,7 @@ pub struct RegisteredScenario { pub fn all() -> Vec { vec![ register("crash-recovery-507", crash_recovery_507::scenario()), - register_console("console-streamed-text", console_streamed_text::scenario()), + register_observe("console-streamed-text", console_streamed_text::scenario()), register("exactly-once-function", exactly_once_function::scenario()), register("hold-mutation-505", hold_mutation_505::scenario()), register("hook-held-release-506", hook_held_release_506::scenario()), @@ -65,11 +65,11 @@ fn register(slug: &str, scenario: builder::Scenario) -> RegisteredScenario { } } -fn register_console(slug: &str, scenario: builder::Scenario) -> RegisteredScenario { +fn register_observe(slug: &str, scenario: builder::Scenario) -> RegisteredScenario { RegisteredScenario { slug: slug.to_string(), authored: scenario.authored, - driver: ScenarioDriver::Console, + driver: ScenarioDriver::Observe, verify: scenario.verify, } } diff --git a/harness/evals/integration/src/scripted_router.rs b/harness/evals/integration/src/scripted_router.rs index d0e58d31d..39a6da3d1 100644 --- a/harness/evals/integration/src/scripted_router.rs +++ b/harness/evals/integration/src/scripted_router.rs @@ -261,23 +261,82 @@ impl ScriptedRouter { } fn with_router_contract(registration: RegisterFunction, function_id: &str) -> RegisterFunction { - let contract = crate::readiness::router_contract(function_id); + let surface = router_surface(function_id); registration - .description( - contract - .description - .expect("router golden must declare a description"), - ) - .request_format( - contract - .request_schema - .expect("router golden must declare a request schema"), - ) - .response_format( - contract - .response_schema - .expect("router golden must declare a response schema"), - ) + .description(surface.description) + .request_format(surface.request_schema) + .response_format(surface.response_schema) +} + +struct RouterFunctionSurface { + description: String, + request_schema: Value, + response_schema: Value, +} + +fn router_surface(function_id: &str) -> RouterFunctionSurface { + router_surfaces() + .into_iter() + .find(|(id, _)| id == function_id) + .map(|(_, surface)| surface) + .unwrap_or_else(|| panic!("no scripted-router golden for {function_id}")) +} + +fn router_surfaces() -> Vec<(String, RouterFunctionSurface)> { + [ + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../llm-router/tests/golden/schemas/router.chat.json" + )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../llm-router/tests/golden/schemas/router.abort.json" + )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../llm-router/tests/golden/schemas/router.models.list.json" + )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../llm-router/tests/golden/schemas/router.models.get.json" + )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../llm-router/tests/golden/schemas/router.models.supports.json" + )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../llm-router/tests/golden/schemas/router.system_prompt.get.json" + )), + ] + .into_iter() + .map(router_surface_from_golden) + .collect() +} + +fn router_surface_from_golden(raw: &str) -> (String, RouterFunctionSurface) { + let value: Value = serde_json::from_str(raw).expect("checked-in function golden is JSON"); + let function_id = value + .get("function_id") + .and_then(Value::as_str) + .expect("router golden must declare function_id") + .to_string(); + let surface = RouterFunctionSurface { + description: value + .get("description") + .and_then(Value::as_str) + .expect("router golden must declare a description") + .to_string(), + request_schema: value + .get("request_schema") + .cloned() + .expect("router golden must declare a request schema"), + response_schema: value + .get("response_schema") + .cloned() + .expect("router golden must declare a response schema"), + }; + (function_id, surface) } fn fixture_model(state: &Arc>) -> ModelFixtureV1 { diff --git a/harness/evals/integration/src/stack.rs b/harness/evals/integration/src/stack.rs index 6728f8c1c..9384eb186 100644 --- a/harness/evals/integration/src/stack.rs +++ b/harness/evals/integration/src/stack.rs @@ -16,12 +16,5 @@ mod tests; pub use crate::process::EarlyExit; pub use bins::StackBins; -pub use config::{ - expected_config_entries, expected_harness_config_entry, render_engine_yaml, render_seed, - WORKER_START_ORDER, -}; pub use layout::RunLayout; pub use supervisor::{free_loopback_port, Stack, StackBootFailure}; - -/// Compatibility name retained for callers of the original stack API. -pub type RunPaths = RunLayout; diff --git a/harness/evals/integration/src/stack/bins.rs b/harness/evals/integration/src/stack/bins.rs index 5dde85a7f..47a69e886 100644 --- a/harness/evals/integration/src/stack/bins.rs +++ b/harness/evals/integration/src/stack/bins.rs @@ -1,14 +1,12 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; -use super::WORKER_START_ORDER; +use super::config::WORKER_START_ORDER; #[derive(Debug, Clone)] pub struct StackBins { pub engine: PathBuf, pub harness: PathBuf, - /// Present only for `serve`, which boots the production Console binary. - pub console: Option, /// queue, iii-directory, session-manager, context-manager. pub workers: BTreeMap, } @@ -19,7 +17,6 @@ impl StackBins { match name { "engine" => Some(&self.engine), "harness" => Some(&self.harness), - "console" => self.console.as_deref(), other => self.workers.get(other).map(PathBuf::as_path), } } diff --git a/harness/evals/integration/src/stack/config.rs b/harness/evals/integration/src/stack/config.rs index 0d0add0e1..e9079e5e2 100644 --- a/harness/evals/integration/src/stack/config.rs +++ b/harness/evals/integration/src/stack/config.rs @@ -71,19 +71,6 @@ pub fn render_seed(worker: &str, layout: &RunLayout) -> Option { } } -pub fn expected_config_entries(layout: &RunLayout) -> Vec<(String, Value)> { - WORKER_START_ORDER - .iter() - .filter_map(|worker| render_seed(worker, layout).map(|seed| (worker.to_string(), seed))) - .collect() -} - -pub fn expected_harness_config_entry(layout: &RunLayout) -> Vec<(String, Value)> { - render_seed("harness", layout) - .map(|seed| vec![("harness".to_string(), seed)]) - .unwrap_or_default() -} - pub(crate) fn write_engine_config(layout: &RunLayout, port: u16) -> anyhow::Result { let path = layout.engine_config_path(); std::fs::write(&path, render_engine_yaml(layout, port))?; diff --git a/harness/evals/integration/src/stack/layout.rs b/harness/evals/integration/src/stack/layout.rs index 72eca6d40..47c67d12e 100644 --- a/harness/evals/integration/src/stack/layout.rs +++ b/harness/evals/integration/src/stack/layout.rs @@ -3,9 +3,6 @@ use std::path::{Path, PathBuf}; use anyhow::Context; /// All filesystem locations owned by one isolated integration run. -/// -/// Keep the four public fields stable: [`super::RunPaths`] is a compatibility -/// alias for this type. #[derive(Debug, Clone)] pub struct RunLayout { pub root: PathBuf, diff --git a/harness/evals/integration/src/stack/manifest.rs b/harness/evals/integration/src/stack/manifest.rs index 060de0c9a..e484ab95d 100644 --- a/harness/evals/integration/src/stack/manifest.rs +++ b/harness/evals/integration/src/stack/manifest.rs @@ -42,20 +42,14 @@ pub(crate) fn stack_info(bins: &StackBins, layout: &RunLayout, port: u16) -> any }; record("engine", &bins.engine)?; record("harness", &bins.harness)?; - if let Some(console) = &bins.console { - record("console", console)?; - } for (name, path) in &bins.workers { record(name, path)?; } - let mut external_workers = WORKER_START_ORDER + let external_workers = WORKER_START_ORDER .iter() .copied() .chain(std::iter::once("harness")) .collect::>(); - if bins.console.is_some() { - external_workers.push("console"); - } Ok(json!({ "profile": STACK_PROFILE, "components": { diff --git a/harness/evals/integration/src/stack/supervisor.rs b/harness/evals/integration/src/stack/supervisor.rs index 315ca92f5..62ca25604 100644 --- a/harness/evals/integration/src/stack/supervisor.rs +++ b/harness/evals/integration/src/stack/supervisor.rs @@ -120,21 +120,6 @@ impl Stack { self.spawn_worker("harness", &bins.harness) } - pub fn spawn_console(&mut self, bins: &StackBins, http_port: u16) -> anyhow::Result<()> { - let bin = bins - .console - .as_deref() - .ok_or_else(|| anyhow::anyhow!("stack has no console binary"))?; - let args = vec![ - "--url".to_string(), - self.ws_url.clone(), - "--http-port".to_string(), - http_port.to_string(), - ]; - let root = self.paths.root.clone(); - self.spawn_child("console", bin, &args, &root) - } - pub async fn kill_engine(&mut self) -> anyhow::Result<()> { let mut engine = self .processes @@ -190,10 +175,6 @@ impl Stack { self.processes.set_teardown_budget(teardown_budget); } - pub fn teardown_budget(&self) -> Duration { - self.processes.teardown_budget() - } - #[doc(hidden)] pub fn spawn_child( &mut self, diff --git a/harness/evals/integration/src/stack/tests.rs b/harness/evals/integration/src/stack/tests.rs index 643c8d08c..56ede651a 100644 --- a/harness/evals/integration/src/stack/tests.rs +++ b/harness/evals/integration/src/stack/tests.rs @@ -51,7 +51,6 @@ fn manifest_is_canonical_and_uses_layout_paths() { let bins = StackBins { engine: binary.clone(), harness: binary.clone(), - console: None, workers: BTreeMap::from([("queue".to_string(), PathBuf::from(&binary))]), }; @@ -76,7 +75,6 @@ fn manifest_fails_when_a_binary_cannot_be_identified() { let bins = StackBins { engine: missing.clone(), harness: missing.clone(), - console: None, workers: BTreeMap::new(), }; let error = manifest::stack_info(&bins, &layout, 3210).unwrap_err(); @@ -90,7 +88,6 @@ async fn boot_failure_carries_a_complete_typed_teardown() { let bins = StackBins { engine: PathBuf::from("/bin/true"), harness: PathBuf::from("/bin/true"), - console: None, workers: BTreeMap::new(), }; diff --git a/harness/evals/integration/src/types/scenario/result.rs b/harness/evals/integration/src/types/scenario/result.rs index 85cdd5ae7..8413aa640 100644 --- a/harness/evals/integration/src/types/scenario/result.rs +++ b/harness/evals/integration/src/types/scenario/result.rs @@ -55,8 +55,8 @@ pub struct IntegrationResultV1 { pub scenario_id: String, pub classification: Classification, /// First floor or verify failure, with run/session/turn ids scrubbed to - /// `{{run_id}}`/`{{session_id}}`/`{{turn_id}}` placeholders so `--repeat` - /// stays byte-stable. + /// `{{run_id}}`/`{{session_id}}`/`{{turn_id}}` placeholders so the + /// persisted result stays comparable across runs. pub failure: Option, pub artifacts: Vec, } diff --git a/harness/evals/integration/src/types/script.rs b/harness/evals/integration/src/types/script.rs index 2a187a6d6..58c7b2c00 100644 --- a/harness/evals/integration/src/types/script.rs +++ b/harness/evals/integration/src/types/script.rs @@ -28,19 +28,13 @@ pub enum SchemaVersion1 { pub struct ModelFixtureV1 { pub id: String, pub provider: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub display_name: Option, pub context_window: u64, pub max_output_tokens: u64, #[serde(skip_serializing_if = "Option::is_none")] - pub input_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub supports_thinking: Option, #[serde(skip_serializing_if = "Option::is_none")] pub supports_xhigh: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_efforts: Option>, - #[serde(skip_serializing_if = "Option::is_none")] pub supports_tools: Option, #[serde(skip_serializing_if = "Option::is_none")] pub supports_vision: Option, @@ -48,31 +42,6 @@ pub struct ModelFixtureV1 { pub supports_cache: Option, #[serde(skip_serializing_if = "Option::is_none")] pub supports_structured_output: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub thinking_budgets: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub pricing: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct ReasoningEffortV1 { - pub effort: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct PricingV1 { - #[serde(skip_serializing_if = "Option::is_none")] - pub input: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_write: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -104,16 +73,12 @@ pub struct JsonNormalizerV1 { /// RFC 6901 JSON Pointer. pub pointer: String, pub operation: NormalizerOperation, - /// Required for `replace`; forbidden for `delete`. - #[serde(skip_serializing_if = "Option::is_none")] - pub replacement: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum NormalizerOperation { Delete, - Replace, } /// The 12 router-request fields; compiled scenarios always make every diff --git a/harness/evals/integration/tests/determinism.rs b/harness/evals/integration/tests/determinism.rs index 0b5ee6b7c..bff0f6590 100644 --- a/harness/evals/integration/tests/determinism.rs +++ b/harness/evals/integration/tests/determinism.rs @@ -1,6 +1,6 @@ -//! `--repeat` requires byte-identical stable results. Failure text is the -//! only run-dependent part of `result.json`, so the scrub must replace every -//! run-scoped id with its placeholder before the result is persisted. +//! Failure text is the only run-dependent part of `result.json`, so the scrub +//! must replace every run-scoped id with its placeholder before the result is +//! persisted. use harness_integration::canonical::canonical_json_pretty; use harness_integration::evidence_data::RunEvidence; diff --git a/harness/evals/integration/tests/readiness.rs b/harness/evals/integration/tests/readiness.rs deleted file mode 100644 index a70d57b21..000000000 --- a/harness/evals/integration/tests/readiness.rs +++ /dev/null @@ -1,171 +0,0 @@ -//! Readiness failure vectors (hidden internal functions, missing context -//! manager, wrong queue topic, schema/seed mismatch) exercised over fake catalogs — the pure checks are the same -//! code the live probe runs. - -use harness_integration::readiness::{ - config_failure, has_function, has_registered_trigger, missing_functions, missing_trigger_types, - registered_trigger_failures, topic_failures, ExpectedTriggerBinding, ReadinessSpec, -}; -use serde_json::json; - -fn spec() -> ReadinessSpec { - ReadinessSpec::harness_surface(vec![( - "harness".to_string(), - json!({ "default_filesystem_root": "off" }), - )]) -} - -#[test] -fn internal_functions_hidden_from_a_filtered_catalog_are_reported() { - // A default (filtered) catalog omits internal ids like harness::send; - // the probe must name the missing surface rather than pass vacuously. - let filtered = json!({ "functions": [ { "function_id": "harness::status" } ] }); - let missing = missing_functions(&spec(), &filtered); - assert_eq!(missing, vec!["function harness::send"]); -} - -#[test] -fn function_catalog_requires_every_structured_id() { - let required = ["recorder::target", "recorder::secondary"]; - let complete = json!({ "functions": [ - { "function_id": "recorder::target" }, - { "id": "recorder::secondary" } - ]}); - assert!(required.iter().all(|id| has_function(&complete, id))); - - let incidental_text = json!({ "functions": [ - { - "function_id": "recorder::target", - "description": "calls recorder::secondary" - } - ]}); - assert!(!required.iter().all(|id| has_function(&incidental_text, id))); -} - -#[test] -fn registered_trigger_catalog_requires_every_bound_function_id() { - let required = ["integration-recorder::lifecycle", "recorder::binding"]; - let complete = json!([ - { "function_id": "integration-recorder::lifecycle" }, - { "function_id": "recorder::binding" } - ]); - assert!(required - .iter() - .all(|id| has_registered_trigger(&complete, id))); - - let incomplete = json!([ - { "function_id": "integration-recorder::lifecycle" } - ]); - assert!(!required - .iter() - .all(|id| has_registered_trigger(&incomplete, id))); -} - -#[test] -fn registered_trigger_contract_requires_exact_type_config_and_cardinality() { - let expected = vec![ExpectedTriggerBinding { - trigger_type: "harness::turn-completed".into(), - function_id: "integration-recorder::lifecycle".into(), - config: json!({ "session_id": "s_1" }), - }]; - let exact = json!({ "registered_triggers": [{ - "id": "trigger-1", - "trigger_type": "harness::turn-completed", - "function_id": "integration-recorder::lifecycle", - "worker_name": "integration-recorder", - "config": { "session_id": "s_1" }, - "config_summary": "{}" - }]}); - assert!(registered_trigger_failures(&expected, &exact).is_empty()); - - let wrong_session = json!({ "registered_triggers": [{ - "id": "trigger-1", - "trigger_type": "harness::turn-completed", - "function_id": "integration-recorder::lifecycle", - "worker_name": "integration-recorder", - "config": { "session_id": "s_other" }, - "config_summary": "{}" - }]}); - assert!(!registered_trigger_failures(&expected, &wrong_session).is_empty()); - - let duplicate = json!({ "registered_triggers": [ - exact["registered_triggers"][0].clone(), - { - "id": "trigger-2", - "trigger_type": "harness::turn-completed", - "function_id": "integration-recorder::lifecycle", - "worker_name": "integration-recorder", - "config": { "session_id": "s_1" }, - "config_summary": "{}" - } - ]}); - assert!(!registered_trigger_failures(&expected, &duplicate).is_empty()); -} - -#[test] -fn missing_context_manager_names_both_functions() { - let spec = ReadinessSpec::pre_harness(vec![]); - let listed = json!({ "functions": [ - { "function_id": "session::messages" }, - { "function_id": "router::chat" }, - { "function_id": "router::abort" }, - { "function_id": "router::models::list" }, - { "function_id": "router::models::get" }, - { "function_id": "router::models::supports" }, - { "function_id": "router::system_prompt::get" }, - { "function_id": "integration-recorder::lifecycle" }, - { "function_id": "engine::queue::list_topics" } - ]}); - let missing = missing_functions(&spec, &listed); - assert_eq!( - missing, - vec![ - "function context::assemble", - "function context::count-tokens" - ] - ); -} - -#[test] -fn wrong_or_absent_queue_topic_is_reported_with_broker_detail() { - let s = spec(); - let empty = topic_failures(&s, &json!([])); - assert_eq!(empty, vec!["queue topic harness-turn"]); - - let wrong_broker = topic_failures( - &s, - &json!([{ "name": "harness-turn", "broker_type": "rabbitmq", "subscriber_count": 1 }]), - ); - assert_eq!( - wrong_broker, - vec!["queue topic harness-turn broker type: expected builtin, got rabbitmq"] - ); - - let ok = topic_failures( - &s, - &json!([{ "name": "harness-turn", "broker_type": "builtin", "subscriber_count": 1 }]), - ); - assert!(ok.is_empty()); -} - -#[test] -fn missing_trigger_type_is_reported() { - let missing = missing_trigger_types(&spec(), &json!([{ "id": "harness::turn-started" }])); - assert_eq!(missing, vec!["trigger type harness::turn-completed"]); -} - -#[test] -fn seed_mismatch_is_reported_but_resolved_defaults_are_tolerated() { - let expected = json!({ "default_filesystem_root": "off" }); - // Worker stored the seed merged with its defaults: seeded key wins → ok. - let resolved = json!({ "value": { "default_filesystem_root": "off", "max_turns": 500 } }); - assert!(config_failure("harness", &expected, &resolved).is_none()); - - // Stored value contradicts the seed → named failure. - let overridden = json!({ "value": { "default_filesystem_root": "/somewhere" } }); - let failure = config_failure("harness", &expected, &overridden).expect("must fail"); - assert!(failure.contains("configuration harness"), "{failure}"); - - // No stored value at all. - assert!(config_failure("harness", &expected, &json!({})).is_some()); -} diff --git a/harness/evals/integration/tests/supervisor.rs b/harness/evals/integration/tests/supervisor.rs index 5bbd8b83e..2943dbce9 100644 --- a/harness/evals/integration/tests/supervisor.rs +++ b/harness/evals/integration/tests/supervisor.rs @@ -4,10 +4,10 @@ use std::path::PathBuf; use std::time::Duration; -use harness_integration::stack::{RunPaths, Stack}; +use harness_integration::stack::{RunLayout, Stack}; -fn stub_stack(dir: &tempfile::TempDir) -> (Stack, RunPaths) { - let paths = RunPaths::allocate(dir.path(), "test-run").expect("allocate"); +fn stub_stack(dir: &tempfile::TempDir) -> (Stack, RunLayout) { + let paths = RunLayout::allocate(dir.path(), "test-run").expect("allocate"); ( Stack::for_tests_with_teardown_budget(paths.clone(), Duration::from_millis(400)), paths, From d63255e857ea2a935a87b4a8e967b20f557c1bcb Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Wed, 22 Jul 2026 07:56:01 -0300 Subject: [PATCH 2/5] (MOT-4107) refactor(integration): focus conformance suite on core flows --- console/web/e2e/durable-hydration.spec.ts | 2 +- console/web/e2e/exactly-once-function.spec.ts | 2 +- console/web/e2e/harness-stack.ts | 254 +++--------- console/web/e2e/ui-send.spec.ts | 6 +- harness/Makefile | 31 +- harness/evals/integration/Cargo.toml | 2 +- harness/evals/integration/README.md | 237 +++++------ .../evals/integration/src/evidence_data.rs | 118 +----- harness/evals/integration/src/expand.rs | 124 +----- .../evals/integration/src/expand/functions.rs | 315 -------------- .../evals/integration/src/expand/render.rs | 25 -- .../evals/integration/src/expand/router.rs | 352 ---------------- .../evals/integration/src/expand/templates.rs | 147 ------- harness/evals/integration/src/expand/tests.rs | 312 -------------- .../integration/src/expand/validation.rs | 162 -------- harness/evals/integration/src/fixtures.rs | 3 - .../integration/src/fixtures/discovery.rs | 74 +--- .../evals/integration/src/fixtures/loading.rs | 114 +++--- .../src/fixtures/script_validation.rs | 248 ----------- .../src/fixtures/stream_validation.rs | 343 ---------------- .../evals/integration/src/fixtures/tests.rs | 375 ++--------------- harness/evals/integration/src/main.rs | 90 ++-- harness/evals/integration/src/matcher.rs | 44 +- .../evals/integration/src/process/child.rs | 27 -- .../integration/src/process/supervisor.rs | 1 + .../evals/integration/src/process/tests.rs | 20 - .../evals/integration/src/recorder/service.rs | 135 +----- .../evals/integration/src/recorder/tests.rs | 23 +- harness/evals/integration/src/runtime.rs | 6 - harness/evals/integration/src/scenario.rs | 13 +- .../evals/integration/src/scenario/floor.rs | 2 +- .../integration/src/scenario/phases/arm.rs | 20 - .../src/scenario/phases/completion.rs | 10 +- .../src/scenario/phases/execution.rs | 345 +--------------- .../scenario/{observe.rs => playground.rs} | 310 ++++++++------ .../evals/integration/src/scenario/runner.rs | 13 +- .../evals/integration/src/scenario/state.rs | 4 + .../integration/src/scenarios/builder.rs | 387 ------------------ .../src/scenarios/console_streamed_text.rs | 94 ++++- .../src/scenarios/crash_recovery_507.rs | 60 --- .../src/scenarios/exactly_once_function.rs | 142 ++++++- .../src/scenarios/hold_mutation_505.rs | 84 ---- .../src/scenarios/hook_held_release_506.rs | 101 ----- .../evals/integration/src/scenarios/mod.rs | 115 +----- .../src/scenarios/streamed_text.rs | 108 +++-- .../integration/src/scenarios/support.rs | 314 ++++++++++++++ harness/evals/integration/src/stack/bins.rs | 2 + .../evals/integration/src/stack/manifest.rs | 3 + .../evals/integration/src/stack/supervisor.rs | 47 +-- harness/evals/integration/src/stack/tests.rs | 3 + .../evals/integration/src/types/recorder.rs | 12 - .../evals/integration/src/types/scenario.rs | 14 +- .../src/types/scenario/authored.rs | 184 --------- .../src/types/scenario/compiled.rs | 48 ++- harness/evals/integration/src/types/script.rs | 1 - .../integration/tests/scenario_compilation.rs | 14 +- harness/evals/integration/tests/schemas.rs | 44 +- 57 files changed, 1277 insertions(+), 4809 deletions(-) delete mode 100644 harness/evals/integration/src/expand/functions.rs delete mode 100644 harness/evals/integration/src/expand/render.rs delete mode 100644 harness/evals/integration/src/expand/router.rs delete mode 100644 harness/evals/integration/src/expand/templates.rs delete mode 100644 harness/evals/integration/src/expand/tests.rs delete mode 100644 harness/evals/integration/src/expand/validation.rs delete mode 100644 harness/evals/integration/src/fixtures/script_validation.rs delete mode 100644 harness/evals/integration/src/fixtures/stream_validation.rs rename harness/evals/integration/src/scenario/{observe.rs => playground.rs} (55%) delete mode 100644 harness/evals/integration/src/scenarios/builder.rs delete mode 100644 harness/evals/integration/src/scenarios/crash_recovery_507.rs delete mode 100644 harness/evals/integration/src/scenarios/hold_mutation_505.rs delete mode 100644 harness/evals/integration/src/scenarios/hook_held_release_506.rs create mode 100644 harness/evals/integration/src/scenarios/support.rs delete mode 100644 harness/evals/integration/src/types/scenario/authored.rs diff --git a/console/web/e2e/durable-hydration.spec.ts b/console/web/e2e/durable-hydration.spec.ts index 5b7829cb0..77bb2d2f7 100644 --- a/console/web/e2e/durable-hydration.spec.ts +++ b/console/web/e2e/durable-hydration.spec.ts @@ -7,7 +7,7 @@ test('hydrates a durable transcript again after a page reload', async ({ stack, }) => { const completed = stack.waitForTurnCompleted() - await stack.start() + await stack.trigger() expect(await completed).toMatchObject({ status: 'completed' }) await openSession(page, stack) diff --git a/console/web/e2e/exactly-once-function.spec.ts b/console/web/e2e/exactly-once-function.spec.ts index 6e56b62c9..1223dbf84 100644 --- a/console/web/e2e/exactly-once-function.spec.ts +++ b/console/web/e2e/exactly-once-function.spec.ts @@ -7,7 +7,7 @@ test('renders one completed function call and its durable result', async ({ stack, }) => { const completed = stack.waitForTurnCompleted() - await stack.start() + await stack.trigger() expect(await completed).toMatchObject({ status: 'completed' }) await openSession(page, stack) diff --git a/console/web/e2e/harness-stack.ts b/console/web/e2e/harness-stack.ts index 54b0b2d57..dcd6b84c4 100644 --- a/console/web/e2e/harness-stack.ts +++ b/console/web/e2e/harness-stack.ts @@ -1,6 +1,5 @@ import { type ChildProcess, spawn } from 'node:child_process' -import { createServer } from 'node:net' -import { mkdir, mkdtemp, readFile, rename, rm, writeFile, watch } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, watch } from 'node:fs/promises' import path from 'node:path' import { setTimeout as delay } from 'node:timers/promises' import type { Page } from '@playwright/test' @@ -12,10 +11,11 @@ interface ReadyManifest { run_id: string scenario_id: string scenario_slug: string - driver: 'direct' | 'observe' + driver: 'direct' | 'playground' run_root: string result_path: string engine_url: string + console_url: string session: { id: string; title: string } model: { id: string; provider: string } message: string @@ -33,7 +33,6 @@ export interface RecorderEvent { received_at: string } -/** Raw serialized RunEvidence: real ids, checkable against ReadyManifest. */ export interface RunEvidence { run_id: string session_id: string @@ -46,7 +45,7 @@ export interface RunEvidence { recorder_events: RecorderEvent[] } -export interface ObserveResult { +export interface PlaygroundResult { schema_version: '1' scenario_id: string classification: @@ -71,9 +70,9 @@ interface TurnCompletedEvent { export interface HarnessStack { ready: ReadyManifest consoleUrl: string - start(): Promise + trigger(): Promise waitForTurnCompleted(): Promise - finish(): Promise + finish(): Promise } interface FixtureOptions { @@ -99,39 +98,17 @@ function workerArgs(): string[] { ].flatMap(([name, env]) => ['--worker-bin', `${name}=${required(env)}`]) } -async function freeLoopbackPort(): Promise { - return await new Promise((resolve, reject) => { - const server = createServer() - server.listen(0, '127.0.0.1', () => { - const address = server.address() - if (!address || typeof address === 'string') { - server.close() - reject(new Error('failed to allocate loopback port')) - return - } - const { port } = address - server.close((error) => { - if (error) reject(error) - else resolve(port) - }) - }) - server.on('error', reject) +function childExit( + child: ChildProcess, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + return new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })) }) } -async function writeAtomicJson(filePath: string, value: unknown): Promise { - const parent = path.dirname(filePath) - const temporary = path.join( - parent, - `.${path.basename(filePath)}.${process.pid}.tmp`, - ) - await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, 'utf8') - await rename(temporary, filePath) -} - async function waitForReady( readyFile: string, - childExit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, + exit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, ): Promise { const read = async (): Promise => { try { @@ -145,20 +122,19 @@ async function waitForReady( const existing = await read() if (existing) return existing - const parent = path.dirname(readyFile) - const expectedName = path.basename(readyFile) - const changes = watch(parent) + const changes = watch(path.dirname(readyFile)) const timeout = delay(70_000).then(() => { throw new Error(`timed out waiting for ${readyFile}`) }) - const exited = childExit.then(({ code, signal }) => { + const exited = exit.then(({ code, signal }) => { throw new Error( `harness-integration exited before ready (code=${String(code)}, signal=${String(signal)})`, ) }) const appeared = (async () => { for await (const event of changes) { - if (event.filename && event.filename !== expectedName) continue + if (event.filename && event.filename !== path.basename(readyFile)) + continue const manifest = await read() if (manifest) return manifest } @@ -171,32 +147,6 @@ async function waitForReady( } } -async function waitForConsoleHttp(port: number): Promise { - const deadline = Date.now() + 60_000 - while (Date.now() < deadline) { - try { - const response = await fetch(`http://127.0.0.1:${port}/`, { - redirect: 'manual', - }) - if (response.ok || (response.status >= 300 && response.status < 400)) { - return - } - } catch { - // Console still booting. - } - await delay(100) - } - throw new Error(`console HTTP did not become ready on port ${port}`) -} - -function childExit( - child: ChildProcess, -): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { - return new Promise((resolve) => { - child.once('exit', (code, signal) => resolve({ code, signal })) - }) -} - function armCompletion( sdk: ISdk, ready: ReadyManifest, @@ -209,16 +159,12 @@ function armCompletion( if (timer) clearTimeout(timer) try { triggerRef?.unregister() - } catch { - // The stack may already be shutting down. - } - try { functionRef?.unregister() } catch { - // The stack may already be shutting down. + // The isolated stack may already be shutting down. } } - const completed = new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { functionRef = sdk.registerFunction( functionId, async (payload) => { @@ -240,13 +186,6 @@ function armCompletion( reject(new Error('harness::turn-completed was not delivered')) }, 60_000) }) - return completed -} - -function stopChild(child: ChildProcess | undefined): void { - if (!child) return - if (child.exitCode !== null || child.signalCode !== null) return - child.kill('SIGTERM') } export const test = base.extend({ @@ -260,15 +199,16 @@ export const test = base.extend({ await mkdir(artifactsRoot, { recursive: true }) const controlDir = await mkdtemp(path.join(artifactsRoot, 'runner-')) const readyFile = path.join(controlDir, 'ready.json') - const startFile = path.join(controlDir, 'start.json') const args = [ - 'observe', + 'playground', '--scenario', scenario, '--engine-bin', required('III_BIN'), '--harness-bin', required('HARNESS_BIN'), + '--console-bin', + required('CONSOLE_BIN'), '--artifacts-dir', artifactsRoot, '--ready-file', @@ -276,7 +216,7 @@ export const test = base.extend({ ...workerArgs(), ] const child = spawn(required('HARNESS_INTEGRATION_BIN'), args, { - stdio: ['pipe', 'pipe', 'pipe'], + stdio: ['ignore', 'pipe', 'pipe'], }) const exit = childExit(child) const stdout: Buffer[] = [] @@ -285,30 +225,24 @@ export const test = base.extend({ child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)) let sdk: ISdk | undefined - let consoleChild: ChildProcess | undefined - let consoleExit: Promise<{ - code: number | null - signal: NodeJS.Signals | null - }> | undefined - const consoleStdout: Buffer[] = [] - const consoleStderr: Buffer[] = [] - let finalized: Promise | undefined - - const finish = (): Promise => { + let ready: ReadyManifest | undefined + let finalized: Promise | undefined + const attachLogs = async () => { + await testInfo.attach('harness-integration.stdout', { + body: Buffer.concat(stdout), + contentType: 'text/plain', + }) + await testInfo.attach('harness-integration.stderr', { + body: Buffer.concat(stderr), + contentType: 'text/plain', + }) + } + const finish = (): Promise => { if (finalized) return finalized finalized = (async () => { + if (!ready) + throw new Error('playground did not publish a ready manifest') if (sdk) await sdk.shutdown().catch(() => undefined) - stopChild(consoleChild) - if (consoleExit) { - let consoleExited = await Promise.race([ - consoleExit, - delay(10_000).then(() => null), - ]) - if (!consoleExited && consoleChild) { - consoleChild.kill('SIGKILL') - consoleExited = await consoleExit - } - } if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM') } @@ -317,28 +251,11 @@ export const test = base.extend({ child.kill('SIGKILL') exited = await exit } - await testInfo.attach('harness-integration.stdout', { - body: Buffer.concat(stdout), - contentType: 'text/plain', - }) - await testInfo.attach('harness-integration.stderr', { - body: Buffer.concat(stderr), - contentType: 'text/plain', - }) - if (consoleStdout.length > 0 || consoleStderr.length > 0) { - await testInfo.attach('console.stdout', { - body: Buffer.concat(consoleStdout), - contentType: 'text/plain', - }) - await testInfo.attach('console.stderr', { - body: Buffer.concat(consoleStderr), - contentType: 'text/plain', - }) - } + await attachLogs() const result = JSON.parse( await readFile(ready.result_path, 'utf8'), - ) as ObserveResult - await testInfo.attach('observe-result', { + ) as PlaygroundResult + await testInfo.attach('playground-result', { body: JSON.stringify(result, null, 2), contentType: 'application/json', }) @@ -347,82 +264,32 @@ export const test = base.extend({ return finalized } - let ready!: ReadyManifest - let consoleUrl!: string try { ready = await waitForReady(readyFile, exit) - const httpPort = await freeLoopbackPort() - consoleUrl = `http://127.0.0.1:${httpPort}` - const spawnedConsole = spawn( - required('CONSOLE_BIN'), - ['--url', ready.engine_url, '--http-port', String(httpPort)], - { stdio: ['ignore', 'pipe', 'pipe'] }, - ) - consoleChild = spawnedConsole - consoleExit = childExit(spawnedConsole) - spawnedConsole.stdout.on('data', (chunk: Buffer) => - consoleStdout.push(chunk), - ) - spawnedConsole.stderr.on('data', (chunk: Buffer) => - consoleStderr.push(chunk), - ) - await Promise.race([ - waitForConsoleHttp(httpPort), - consoleExit.then(({ code, signal }) => { - throw new Error( - `console exited before ready (code=${String(code)}, signal=${String(signal)})`, - ) - }), - ]) + const manifest = ready + const connectedSdk = registerWorker(manifest.engine_url) + sdk = connectedSdk + const stack: HarnessStack = { + ready: manifest, + consoleUrl: manifest.console_url, + trigger: () => + connectedSdk.trigger({ + function_id: 'harness::send', + payload: manifest.send, + }), + waitForTurnCompleted: () => armCompletion(connectedSdk, manifest), + finish, + } + await use(stack) } catch (error) { - stopChild(consoleChild) if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM') } await exit.catch(() => undefined) - await consoleExit?.catch(() => undefined) - await testInfo.attach('harness-integration.stdout', { - body: Buffer.concat(stdout), - contentType: 'text/plain', - }) - await testInfo.attach('harness-integration.stderr', { - body: Buffer.concat(stderr), - contentType: 'text/plain', - }) - if (consoleStdout.length > 0 || consoleStderr.length > 0) { - await testInfo.attach('console.stdout', { - body: Buffer.concat(consoleStdout), - contentType: 'text/plain', - }) - await testInfo.attach('console.stderr', { - body: Buffer.concat(consoleStderr), - contentType: 'text/plain', - }) - } + await attachLogs() throw error - } - - const connectedSdk = registerWorker(ready.engine_url) - sdk = connectedSdk - let started = false - const stack: HarnessStack = { - ready, - consoleUrl, - start: async () => { - if (started) return - started = true - await writeAtomicJson(startFile, { schema_version: '1' }) - }, - waitForTurnCompleted: () => armCompletion(connectedSdk, ready), - finish, - } - - try { - await use(stack) } finally { - if (!finalized) { - await finish().catch(() => undefined) - } + if (!finalized && ready) await finish().catch(() => undefined) await rm(controlDir, { recursive: true, force: true }) } }, @@ -435,6 +302,11 @@ export async function openSession( stack: HarnessStack, ): Promise { await page.goto(stack.consoleUrl) + // Let the Console settle its initial local-draft selection before changing + // sessions; otherwise that bootstrap effect can overwrite this click. + await expect( + page.locator('[role="button"][aria-current="page"]'), + ).toHaveCount(1) const session = page.getByRole('button', { name: `open ${stack.ready.session.title}`, exact: true, @@ -443,6 +315,6 @@ export async function openSession( await expect(session).toHaveAttribute('aria-current', 'page') } -export function expectPassingResult(result: ObserveResult): void { +export function expectPassingResult(result: PlaygroundResult): void { expect(result.classification).toBe('pass') } diff --git a/console/web/e2e/ui-send.spec.ts b/console/web/e2e/ui-send.spec.ts index 5955d1768..08979749d 100644 --- a/console/web/e2e/ui-send.spec.ts +++ b/console/web/e2e/ui-send.spec.ts @@ -2,13 +2,15 @@ import { expect, expectPassingResult, openSession, test } from './harness-stack' test.use({ scenario: 'console-streamed-text' }) -test('renders a harness-started streamed turn in the Console', async ({ +test('sends and renders a streamed turn through the Console', async ({ page, stack, }) => { const completed = stack.waitForTurnCompleted() await openSession(page, stack) - await stack.start() + const composer = page.getByLabel('message composer') + await composer.fill(stack.ready.message) + await composer.press('Enter') await expect( page.locator('[data-message-role="user"]', { diff --git a/harness/Makefile b/harness/Makefile index 1ba679b3e..f025781b6 100644 --- a/harness/Makefile +++ b/harness/Makefile @@ -28,7 +28,7 @@ RUST_WORKERS := $(filter-out $(PYTHON_WORKERS),$(STACK)) GOAL_ARGS := $(filter-out help install-iii-next build install-local clean cargo-clean dev-up dev-down dev-restart \ engine wait-engine restart restart-worker stop stop-worker stop-work stop-engine status smoke logs attach \ - ensure-session prepare-scrapling require-engine where integration-e2e integration-validate,$(MAKECMDGOALS)) + ensure-session prepare-scrapling require-engine where integration-e2e integration-playground integration-validate,$(MAKECMDGOALS)) SELECTED_WORKERS := $(strip $(GOAL_ARGS)) ACTIVE_WORKERS := $(if $(SELECTED_WORKERS),$(SELECTED_WORKERS),$(STACK)) UNKNOWN_WORKERS := $(filter-out $(STACK),$(SELECTED_WORKERS)) @@ -40,7 +40,7 @@ RUN_FLAG := $(if $(filter release,$(RUN_PROFILE)),--release,) .PHONY: help install-iii-next build install-local clean cargo-clean dev-up dev-down dev-restart engine wait-engine \ restart restart-worker stop stop-worker stop-work stop-engine status smoke logs attach ensure-session prepare-scrapling \ - require-engine where integration-e2e integration-validate $(STACK) + require-engine where integration-e2e integration-playground integration-validate $(STACK) help: @printf '%s\n' \ @@ -62,6 +62,7 @@ help: ' make cargo-clean [workers...] cargo clean stack or selected workers' \ ' make install-local [workers...] build and symlink into ~/.iii/workers' \ ' make integration-e2e III_BIN= run the harness integration scenarios (evals/integration)' \ + ' make integration-playground III_BIN= open the Console against an isolated integration stack' \ ' make integration-validate validate every integration scenario' install-iii-next: @@ -320,6 +321,8 @@ INTEGRATION_PROFILE ?= release INTEGRATION_FLAG := $(if $(filter release,$(INTEGRATION_PROFILE)),--release,) INTEGRATION_SCENARIO ?= all INTEGRATION_ARTIFACTS ?= $(REPO_ROOT)/target/integration +INTEGRATION_PLAYGROUND_SCENARIO ?= console-streamed-text +INTEGRATION_PLAYGROUND_ARTIFACTS ?= $(REPO_ROOT)/target/console-e2e integration-e2e: @if [ -z "$(III_BIN)" ]; then \ @@ -344,6 +347,30 @@ integration-e2e: --scenario "$(INTEGRATION_SCENARIO)" \ --artifacts-dir "$(INTEGRATION_ARTIFACTS)" +integration-playground: + @if [ -z "$(III_BIN)" ]; then \ + echo "III_BIN is required: path to the pinned iii engine binary."; \ + echo "Pinned source: harness/evals/integration/engine.lock"; \ + exit 3; \ + fi + @for w in $(INTEGRATION_WORKERS) harness console; do \ + echo "building $$w ($(INTEGRATION_PROFILE))"; \ + cargo build $(INTEGRATION_FLAG) --manifest-path "$(REPO_ROOT)/$$w/Cargo.toml" || exit 1; \ + done + @echo "building harness-integration ($(INTEGRATION_PROFILE))" + @cargo build $(INTEGRATION_FLAG) --manifest-path "$(MAKEFILE_DIR)evals/integration/Cargo.toml" + @"$(MAKEFILE_DIR)evals/integration/target/$(INTEGRATION_PROFILE)/harness-integration" \ + playground \ + --engine-bin "$(III_BIN)" \ + --harness-bin "$(REPO_ROOT)/harness/target/$(INTEGRATION_PROFILE)/harness" \ + --console-bin "$(REPO_ROOT)/console/target/$(INTEGRATION_PROFILE)/console" \ + --worker-bin "queue=$(REPO_ROOT)/queue/target/$(INTEGRATION_PROFILE)/queue" \ + --worker-bin "iii-directory=$(REPO_ROOT)/iii-directory/target/$(INTEGRATION_PROFILE)/iii-directory" \ + --worker-bin "session-manager=$(REPO_ROOT)/session-manager/target/$(INTEGRATION_PROFILE)/session-manager" \ + --worker-bin "context-manager=$(REPO_ROOT)/context-manager/target/$(INTEGRATION_PROFILE)/context-manager" \ + --scenario "$(INTEGRATION_PLAYGROUND_SCENARIO)" \ + --artifacts-dir "$(INTEGRATION_PLAYGROUND_ARTIFACTS)" + integration-validate: @cargo run --quiet --manifest-path "$(MAKEFILE_DIR)evals/integration/Cargo.toml" -- \ validate --scenario all diff --git a/harness/evals/integration/Cargo.toml b/harness/evals/integration/Cargo.toml index 1230f26a7..233cb283d 100644 --- a/harness/evals/integration/Cargo.toml +++ b/harness/evals/integration/Cargo.toml @@ -32,13 +32,13 @@ clap = { version = "4", features = ["derive", "env"] } schemars = "0.8" uuid = { version = "1", features = ["v4"] } regex = "1" -jsonschema = { version = "0.18", default-features = false } sha2 = "0.10" time = { version = "0.3", features = ["formatting", "parsing"] } nix = { version = "0.29", features = ["signal", "process"] } [dev-dependencies] tempfile = "3" +jsonschema = { version = "0.18", default-features = false } # stack.json digests every spawned binary (spec: recorded before boot); # debug-mode sha2 turns ~800MB of debug worker binaries into a ~40s stall. diff --git a/harness/evals/integration/README.md b/harness/evals/integration/README.md index 14a92ce69..70c23d946 100644 --- a/harness/evals/integration/README.md +++ b/harness/evals/integration/README.md @@ -1,30 +1,35 @@ # Harness integration E2E -Deterministic public-path regression tests for the harness. Every scenario -boots a fresh isolated stack with the pinned engine and real queue, +Deterministic public-path regression tests for the harness. Each scenario +boots a fresh isolated stack with the pinned engine and the real queue, session-manager, context-manager, iii-directory, and harness workers. Only the `router::*` model boundary is replaced by a strict scripted worker. No provider key or network access is required. -## Run +## Scenarios -```bash -# Build the stack and run every non-quarantined scenario. -make -C harness integration-e2e III_BIN= +| id | slug | driver | coverage | +|---|---|---|---| +| E2E-001 | `streamed-text` | direct | streamed text reaches durable completion | +| E2E-002 | `exactly-once-function` | direct | a native function executes exactly once | +| UI-001 | `console-streamed-text` | playground | a message sent by the Console streams to durable completion | -# Validate every fixture, including quarantined reproductions. -make -C harness integration-validate +Each fixture is defined end to end in its own `src/scenarios/*.rs` file: the +exact `harness::send` payload, router request matchers, response frames, +recorder configuration, and scenario-specific verification function. Shared +code is limited to wire-format constructors. There is no YAML or generic +authored-scenario compiler. -# Run one scenario directly. -harness-integration run \ - --engine-bin \ - --harness-bin \ - --worker-bin queue= \ - --worker-bin session-manager= \ - --worker-bin context-manager= \ - --worker-bin iii-directory= \ - --scenario E2E-001 +## Run the direct scenarios + +```bash +make -C harness integration-e2e III_BIN= + +# Select one direct scenario by id or slug. +make -C harness integration-e2e \ + III_BIN= \ + INTEGRATION_SCENARIO=E2E-001 ``` The engine is never downloaded by the runner. CI builds the source revision @@ -37,143 +42,85 @@ Exit codes are: - `2`: contract failure or scenario timeout; - `3`: setup, process, or runner error. -## Create a scenario - -Each scenario is one Rust module: `src/scenarios/.rs`, one `scenario()` -function that builds the authored stimulus through the typed builders in -`src/scenarios/builder.rs` and closes the chain with `.verify(|run| ...)`, -registered in `src/scenarios/mod.rs`. A scenario without checks does not -typecheck. There is no YAML layer — the authored -shape is enforced by the type system at `cargo build` and is never -serialized. Model/provider, session id, idempotency key, native function -policy, run-scoped function ids, request matchers, response frames, and -system prompt hash are inferred by the compiler. - -A typical authored function scenario is: - -```rust -// src/scenarios/my_function_case.rs -pub(super) fn scenario() -> Scenario { - AuthoredScenario::new("E2E-010", "The allowed function runs once.") - .trigger(Harness::send("Call the recorder once.")) - .function( - "record", - Function::new( - "Record one value.", - json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }), - json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - ), - ) - .model(( - Reply::function_call("record", json!({ "value": "expected" })), - Reply::text("recorded once"), - )) - .verify(|run| { - let calls = run.calls("record"); - anyhow::ensure!(calls.len() == 1, "record ran {} times", calls.len()); - anyhow::ensure!(calls[0].payload == json!({ "value": "expected" })); - anyhow::ensure!(!run.has_duplicate_messages()); - Ok(()) - }) -} +## Open an isolated Console playground + +```bash +make -C harness integration-playground III_BIN= ``` -The scenario returns a dataset; the author writes the checks in plain Rust. -The floor is enforced by the runner before `verify` is called — turn -completed (terminal status, lifecycle delivered exactly once), script fully -consumed (every scripted generation used, none extra), and a clean send — -and any violation is a `contract_failure` with a `floor: ` message. -`verify(run)` then receives the full `RunEvidence` dataset (send response, -final status, transcript, recorder events, router consumption) for -scenario-specific checks; accessors such as `assistant_texts()`, -`message_counts()`, `calls(alias)`, and `all_calls_closed()` cover the -recurring reads. The runner catches panics, so `assert!`/`assert_eq!` are -allowed; prefer `anyhow::ensure!` where a message helps. +The command builds and starts the production Console together with the +isolated integration stack. It creates the scenario session and prints its +Console URL. For the default UI-001 scenario: + +1. Open the printed URL. +2. Select the pre-created integration session. +3. Send `Return the console fixture phrase.` through the message composer. +4. Wait for `console fixture complete`. +5. Stop the command with Ctrl-C. + +After a completed turn, shutdown collects evidence, grades the scenario, and +writes `playground-result.json`. Stopping before a turn completes is a +contract failure. -Add the module and its slug to the list in `src/scenarios/mod.rs`, then: +The underlying command accepts one scenario only: ```bash -cargo test # builder and contract tests -harness-integration validate --scenario all -harness-integration render E2E-010 +harness-integration playground \ + --engine-bin \ + --harness-bin \ + --console-bin \ + --worker-bin queue= \ + --worker-bin session-manager= \ + --worker-bin context-manager= \ + --worker-bin iii-directory= \ + --scenario console-streamed-text +``` + +`--ready-file ` optionally publishes an atomic JSON manifest for +Playwright. The manifest includes the engine and Console URLs, session, +scenario, model, message, controlled function ids, compiled direct send, and +result path. There is no separate start signal: Playwright either invokes the +compiled send through the SDK or submits through the Console UI. + +## Validate fixtures + +```bash +make -C harness integration-validate +cargo test --manifest-path harness/evals/integration/Cargo.toml +cargo clippy --manifest-path harness/evals/integration/Cargo.toml \ + --all-targets -- -D warnings ``` -Builders produce data only — a builder that derives scenario content from -control flow is rejected in review. New scenarios are runnable by -default; chain `.quarantine()` only for a known reproduction that should be -excluded from `run --scenario all`. `render` prints deterministic canonical -JSON with the complete compiled request, router script, and system prompt. - -`Function::recorder()` is the canonical string-in/`recorded`-out fixture; -`Function::new(...)` builds any other controlled function and `.hidden()` -marks a hook-only one. Function aliases become `::`; every -exposed function is dispatchable. `Release::execute()` releases a held call -for execution. Typed text and function-call replies cover normal cases; -`.recovery_boundary()` matches a reply against the durable outcome only, -where a fault restart or hook release may rebuild the request, and -`.match_overrides(...)` is the remaining escape hatch for intentionally -different wire shapes. - -Timeout defaults are 60 seconds for setup waits (e.g. observer start), 60 -seconds for the scenario, and 15 seconds for teardown. The scenario budget -can be raised with `.scenario_timeout_ms(...)` (crash-recovery does). - -## Checked-in scenarios - -| id | slug | status | -|---|---|---| -| E2E-001 | `streamed-text` | streamed text reaches durable completion | -| E2E-002 | `exactly-once-function` | a native function executes exactly once | -| UI-001 | `console-streamed-text` | integration starts a streamed turn; Playwright validates Console UI | -| E2E-505 | `hold-mutation-505` | quarantined reproduction for issue #505 | -| E2E-506 | `hook-held-release-506` | quarantined reproduction for issue #506 | -| E2E-507 | `crash-recovery-507` | quarantined reproduction for issue #507 | - -`run --scenario all` includes non-quarantined direct scenarios. Observe-driven -UI scenarios (and Direct scenarios used from Playwright) run through -`observe --scenario `: the integration publishes `ready.json`, -waits for Playwright's `start.json`, then runs `harness::send` and grades -backend evidence while Playwright owns the Console process and DOM asserts. -An explicit quarantined direct scenario still runs; `validate --scenario all` -always includes every driver and quarantine state. +`validate --scenario all` checks exactly the three fixtures. `run --scenario +all` executes only E2E-001 and E2E-002; UI-001 must use `playground`. + +The fixture tests pin: + +- the streamed frame sequence and terminal response agreement; +- function-call and function-result history for E2E-002; +- the Console-specific system-prompt and `agent_trigger` tool matchers; +- serialization round trips and the authoritative `harness::send` schema. ## Runtime and evidence -The lifecycle is allocate → boot → arm → send → optional fault or release → -await → collect → grade → teardown → report. Observe inserts Probe (wait for -`start.json`) between Arm and Send, then waits for observer shutdown after -Await before Collect. +The direct lifecycle is allocate → boot → arm → send → await → collect → +grade → teardown → report. Playground replaces send with an externally +initiated Console or SDK turn and waits for shutdown after completion. -- All RPCs and polling share monotonic phase deadlines. -- The recorder keeps configuration and snapshots in process; only controlled - target functions and the lifecycle sink are registered with the engine. +- Completion is driven by the `harness::turn-completed` lifecycle event. +- All RPCs and polling use bounded monotonic deadlines. - Recorder acknowledgements happen only after append and `fsync`. -- Child processes run in dedicated process groups and teardown signals the - complete group and direct child with SIGTERM followed by SIGKILL, within - one hard cleanup budget. -- Router and evidence comparisons use explicit JSON array policies. - -Each run writes `result.json`, `execution.json`, `teardown.json`, and -`stack.json` below `target/integration//`; scenario evidence lives -under `scenarios//` (transcript, status, router calls, target -calls, lifecycle events). `result.json` contains the stable byte-comparable -verdict: the classification plus the first floor or verify failure message, -with run/session/turn ids scrubbed to placeholders. `execution.json` contains -the run id, timing, scenario id, and SHA-256 of the exact `result.json` -bytes. In observe mode, `observe-result.json` additionally carries the raw -serialized `RunEvidence` (real ids) so Playwright can check it against the -ready manifest. Passing runs retain the compact reports and remove -heavyweight stack state unless `--retain-success` is supplied. - -The compiler uses the Harness's embedded `prompts/default.txt` directly, -appends the inferred session and function policy, then hashes the result for -strict router matching. A scenario may explicitly override the router's prompt -matcher in its builder without replacing the shared prompt source. +- Child processes run in dedicated process groups and teardown uses SIGTERM + followed by SIGKILL within one hard cleanup budget. +- Router matching is explicit for all request fields. + +Direct runs write `result.json`, `execution.json`, `teardown.json`, and +`stack.json` below `target/integration//`. Scenario evidence includes +the transcript, status, router calls, controlled target calls, and lifecycle +events. `result.json` is stable across runs because concrete run, session, and +turn ids are scrubbed from failure text; `execution.json` records the SHA-256 +of those exact result bytes. + +Playground runs use `target/console-e2e//` and additionally write +`playground-ready.json` and `playground-result.json`. Passing runs keep compact +reports and remove heavyweight stack state. diff --git a/harness/evals/integration/src/evidence_data.rs b/harness/evals/integration/src/evidence_data.rs index 8b8de701a..495c99454 100644 --- a/harness/evals/integration/src/evidence_data.rs +++ b/harness/evals/integration/src/evidence_data.rs @@ -16,7 +16,7 @@ pub struct RunEvidence { pub run_id: String, pub session_id: String, pub turn_id: Option, - /// `harness::send` response — present after Direct/Observe Send succeeds. + /// `harness::send` response — present when the direct runner owns Send. pub send_response: Option, /// Final `harness::status` report (JSON null when the session is unknown). pub status: Value, @@ -89,48 +89,6 @@ impl RunEvidence { .any(|entry_id| !seen.insert(entry_id)) } - /// Every dispatched function call is closed by a durable result. - pub fn all_calls_closed(&self) -> bool { - let mut call_ids = Vec::new(); - let mut result_ids = std::collections::BTreeSet::new(); - for message in self.messages() { - match role(message) { - Some("assistant") => { - for block in message - .get("content") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - if block.get("type").and_then(Value::as_str) == Some("function_call") { - if let Some(id) = block.get("id").and_then(Value::as_str) { - call_ids.push(id); - } - } - } - } - Some("function_result") => { - if let Some(id) = message.get("function_call_id").and_then(Value::as_str) { - result_ids.insert(id); - } - } - _ => {} - } - } - call_ids.iter().all(|id| result_ids.contains(id)) - } - - /// Exactly one durable function result closes the given call id. - pub fn function_result_closes(&self, call_id: &str) -> bool { - self.messages() - .filter(|message| role(message) == Some("function_result")) - .filter(|message| { - message.get("function_call_id").and_then(Value::as_str) == Some(call_id) - }) - .count() - == 1 - } - /// Replace this run's concrete ids with `{{run_id}}` / `{{session_id}}` / /// `{{turn_id}}` placeholders so persisted failure text stays /// byte-comparable across runs. @@ -161,14 +119,6 @@ fn replace_identity(text: &mut String, identity: &str, placeholder: &str) { } } -/// Structural subset check for hook payloads: every `subset` object member -/// must appear in `actual` (extra members allowed), arrays must match -/// element-for-element. -pub fn json_contains(actual: &Value, subset: &Value) -> bool { - crate::matcher::subset_with_array_policy(subset, actual, crate::matcher::ArrayPolicy::Exact) - .is_none() -} - #[cfg(test)] mod tests { use serde_json::json; @@ -247,52 +197,6 @@ mod tests { assert_eq!(evidence.lifecycle_events().len(), 1); } - #[test] - fn dangling_function_calls_fail_calls_closed() { - let mut evidence = base_evidence(); - evidence.transcript = vec![json!({ - "message": { - "role": "assistant", - "content": [{ "type": "function_call", "id": "call-1" }] - } - })]; - assert!(!evidence.all_calls_closed()); - evidence.transcript.push(json!({ - "message": { "role": "function_result", "function_call_id": "call-1" } - })); - assert!(evidence.all_calls_closed()); - } - - #[test] - fn function_result_closes_selects_its_call_id() { - let mut evidence = base_evidence(); - evidence.transcript = vec![ - json!({ - "message": { - "role": "function_result", - "function_call_id": "call-1", - "content": [] - } - }), - json!({ - "message": { - "role": "function_result", - "function_call_id": "call-2", - "content": [] - } - }), - ]; - assert!(evidence.function_result_closes("call-1")); - assert!(evidence.function_result_closes("call-2")); - assert!(!evidence.function_result_closes("call-3")); - - // A second result for the same call is a defect, not a pass. - evidence.transcript.push(json!({ - "message": { "role": "function_result", "function_call_id": "call-1" } - })); - assert!(!evidence.function_result_closes("call-1")); - } - #[test] fn message_counts_and_assistant_texts_read_the_transcript() { let mut evidence = base_evidence(); @@ -322,26 +226,6 @@ mod tests { assert_eq!(evidence.assistant_texts(), ["recorded once"]); } - #[test] - fn json_contains_is_a_recursive_subset_with_exact_arrays() { - let actual = json!({ - "point": "pre_trigger", - "call": { "id": "call-1", "arguments": { "value": "expected" } }, - "extra": true - }); - assert!(json_contains( - &actual, - &json!({ "call": { "arguments": { "value": "expected" } } }) - )); - assert!(!json_contains( - &actual, - &json!({ "call": { "arguments": { "value": "other" } } }) - )); - // Arrays are exact: a missing element cannot hide behind subset laxity. - assert!(!json_contains(&json!([1, 2, 3]), &json!([1, 2]))); - assert!(json_contains(&json!([1, 2]), &json!([1, 2]))); - } - #[test] fn scrub_replaces_every_run_scoped_id() { let mut evidence = base_evidence(); diff --git a/harness/evals/integration/src/expand.rs b/harness/evals/integration/src/expand.rs index 535ffeb7d..c8d9b5bcc 100644 --- a/harness/evals/integration/src/expand.rs +++ b/harness/evals/integration/src/expand.rs @@ -1,32 +1,15 @@ -//! Compilation and placeholder expansion for authored scenarios. -//! -//! Authors work with aliases and typed replies. Compilation produces the -//! exact, strict runtime structures before any process is started. +//! Run-scoped placeholder expansion for the three checked-in fixtures. -mod functions; -mod render; -mod router; -mod templates; mod tokens; -mod validation; - -#[cfg(test)] -mod tests; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::types::scenario::{AuthoredScenarioV1, CompiledScenarioV1}; -use crate::types::script::{ModelFixtureV1, RouterScriptV1, SchemaVersion1}; +use crate::types::scenario::CompiledScenarioV1; +use crate::types::script::RouterScriptV1; -pub use render::render_compiled; -pub use templates::{scenario_template, ScenarioTemplateKind}; pub(crate) use tokens::Placeholders; -const DEFAULT_MODEL: &str = "fixture-model"; -const DEFAULT_PROVIDER: &str = "scripted"; -const SYNTHETIC_FUNCTION_ALIAS: &str = "unused"; - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CompiledFixtureV1 { @@ -70,104 +53,3 @@ pub(crate) fn expand_compiled_fixture( system_prompt, }) } - -#[derive(Debug)] -struct CompiledFunctionCall { - id: String, - function: String, - generation_index: usize, -} - -/// Compile the concise authored contract to the strict structures consumed by -/// the runner and scripted router. -pub fn compile_scenario( - authored: &AuthoredScenarioV1, - system_prompt_base: &str, -) -> anyhow::Result { - validation::validate_identity(authored)?; - let model = authored - .router - .model - .clone() - .unwrap_or_else(default_model_fixture); - if model.id.is_empty() || model.provider.is_empty() { - anyhow::bail!("router model id and provider must be non-empty"); - } - if authored.router.generations.is_empty() { - anyhow::bail!("router has no generations"); - } - - let allowed_aliases = functions::allowed_aliases(authored); - let function_ids = functions::function_ids(authored); - let allowed_ids: Vec = allowed_aliases - .iter() - .map(|alias| function_ids[alias].clone()) - .collect(); - let tools = functions::compile_tools(authored, &allowed_aliases, &function_ids); - let mut recorder = functions::compile_recorder(authored, &allowed_aliases, &function_ids); - let bindings = functions::compile_bindings(authored, &allowed_aliases, &function_ids)?; - let calls = functions::function_call_ids(authored)?; - validation::validate_release(authored, &calls)?; - let fault = functions::compile_fault(authored, &function_ids, &calls)?; - if let Some(fault) = &fault { - functions::hold_fault_target(&mut recorder, &fault.function_id, fault.after_target_calls)?; - } - - let send = functions::compile_send(authored, &model, &allowed_ids)?; - let script = router::compile_router(authored, &model, &tools, &function_ids, &calls)?; - let system_prompt_template = compile_system_prompt(system_prompt_base, &allowed_ids); - - let fixture = CompiledFixtureV1 { - scenario: CompiledScenarioV1 { - schema_version: SchemaVersion1::V1, - id: authored.id.clone(), - description: authored.description.clone(), - send, - recorder, - deadlines: authored.timeouts, - fault, - bindings, - release: authored.release.clone(), - }, - script, - system_prompt_template, - }; - render::validate_placeholders(&fixture)?; - Ok(fixture) -} - -fn compile_system_prompt(base: &str, allowed_ids: &[String]) -> String { - let base = base.strip_suffix('\n').unwrap_or(base); - let policy = if allowed_ids.is_empty() { - "Function dispatch is entirely disabled this turn — do not call any function.".to_string() - } else { - let mut allowed = allowed_ids.to_vec(); - allowed.sort(); - allowed.dedup(); - format!( - "Your dispatch policy allows ONLY these functions: {}. This narrowed-policy \ - instruction OVERRIDES the general discovery requirement for this turn: call the \ - listed target ids directly when the task already supplies their arguments. Anything \ - else — including discovery (engine::functions::list / ::info) unless listed above — \ - is denied. Do not probe: if the task genuinely needs an unlisted function or an \ - unknown contract, report that blocker and finish.", - allowed.join(", ") - ) - }; - format!("{base}\n\nYour session id is {{{{session_id}}}}.\n{policy}") -} - -fn default_model_fixture() -> ModelFixtureV1 { - ModelFixtureV1 { - id: DEFAULT_MODEL.to_string(), - provider: DEFAULT_PROVIDER.to_string(), - context_window: 32_768, - max_output_tokens: 4_096, - supports_thinking: Some(false), - supports_xhigh: None, - supports_tools: Some(true), - supports_vision: Some(false), - supports_cache: Some(false), - supports_structured_output: Some(true), - } -} diff --git a/harness/evals/integration/src/expand/functions.rs b/harness/evals/integration/src/expand/functions.rs deleted file mode 100644 index 9424cb78f..000000000 --- a/harness/evals/integration/src/expand/functions.rs +++ /dev/null @@ -1,315 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; - -use anyhow::Context; -use serde_json::{json, Value}; - -use crate::types::recorder::{ - LifecycleFunctionId, LifecycleTriggerType, RecorderConfigV1, RecorderLifecycleV1, - RecorderTargetV1, -}; -use crate::types::scenario::{ - AuthoredScenarioV1, CompiledFaultV1, RouterReplyV1, ScenarioFunctionV1, TriggerBindingV1, -}; -use crate::types::script::ModelFixtureV1; - -use super::{ - validation::{validate_function_arguments, validate_hook_response}, - CompiledFunctionCall, SYNTHETIC_FUNCTION_ALIAS, -}; - -pub(super) fn function_ids(authored: &AuthoredScenarioV1) -> BTreeMap { - authored - .functions - .keys() - .map(|alias| (alias.clone(), format!("{{{{run_id}}}}::{alias}"))) - .collect() -} - -/// Every exposed function alias, in stable (BTreeMap) order. -pub(super) fn allowed_aliases(authored: &AuthoredScenarioV1) -> Vec { - authored - .functions - .iter() - .filter(|(_, function)| function.expose) - .map(|(alias, _)| alias.clone()) - .collect() -} - -pub(super) fn compile_tools( - authored: &AuthoredScenarioV1, - allowed_aliases: &[String], - function_ids: &BTreeMap, -) -> Value { - Value::Array( - allowed_aliases - .iter() - .map(|alias| { - let function = &authored.functions[alias]; - json!({ - "name": function_ids[alias], - "description": function.description, - "parameters": function.request_schema, - "execution_mode": "sequential" - }) - }) - .collect(), - ) -} - -pub(super) fn compile_recorder( - authored: &AuthoredScenarioV1, - allowed_aliases: &[String], - function_ids: &BTreeMap, -) -> RecorderConfigV1 { - if authored.functions.is_empty() { - return RecorderConfigV1 { - target: synthetic_target(), - lifecycle: compiled_lifecycle(), - extra_functions: Vec::new(), - }; - } - - let target_alias = allowed_aliases - .first() - .cloned() - .or_else(|| authored.functions.keys().next().cloned()) - .expect("non-empty functions has a target"); - let target = compile_function( - &function_ids[&target_alias], - &authored.functions[&target_alias], - ); - let extra_functions = authored - .functions - .iter() - .filter(|(alias, _)| *alias != &target_alias) - .map(|(alias, function)| compile_function(&function_ids[alias], function)) - .collect(); - RecorderConfigV1 { - target, - lifecycle: compiled_lifecycle(), - extra_functions, - } -} - -fn compile_function(function_id: &str, function: &ScenarioFunctionV1) -> RecorderTargetV1 { - RecorderTargetV1 { - function_id: function_id.to_string(), - description: function.description.clone(), - request_schema: function.request_schema.clone(), - response: function.response.clone(), - hold_response_at: None, - } -} - -fn synthetic_target() -> RecorderTargetV1 { - RecorderTargetV1 { - function_id: format!("{{{{run_id}}}}::{SYNTHETIC_FUNCTION_ALIAS}"), - description: "Synthetic integration target; must never be called.".to_string(), - request_schema: json!({ - "type": "object", - "additionalProperties": false - }) - .as_object() - .cloned() - .expect("object"), - response: json!({ - "content": [{ "type": "text", "text": "unused" }], - "is_error": false - }), - hold_response_at: None, - } -} - -fn compiled_lifecycle() -> RecorderLifecycleV1 { - RecorderLifecycleV1 { - trigger_type: LifecycleTriggerType::TurnCompleted, - function_id: LifecycleFunctionId::Lifecycle, - } -} - -pub(super) fn compile_bindings( - authored: &AuthoredScenarioV1, - allowed_aliases: &[String], - function_ids: &BTreeMap, -) -> anyhow::Result> { - authored - .bindings - .iter() - .map(|binding| { - let function_id = function_ids.get(&binding.function).with_context(|| { - format!( - "binding references unknown controlled function alias {:?}", - binding.function - ) - })?; - let callback = &authored.functions[&binding.function]; - if callback.expose { - anyhow::bail!( - "binding callback {:?} must set expose: false", - binding.function - ); - } - validate_hook_response(&binding.function, &callback.response)?; - if binding.functions.is_empty() { - anyhow::bail!( - "binding callback {:?} must select at least one exposed function", - binding.function - ); - } - let mut selected = Vec::with_capacity(binding.functions.len()); - let mut seen = BTreeSet::new(); - for alias in &binding.functions { - let selected_id = function_ids.get(alias).cloned().with_context(|| { - format!("binding references unknown exposed function alias {alias:?}") - })?; - if !seen.insert(alias) { - anyhow::bail!( - "binding callback {:?} selects duplicate function alias {alias:?}", - binding.function - ); - } - if !allowed_aliases.contains(alias) { - anyhow::bail!( - "binding callback {:?} selects function {alias:?} that send does not expose", - binding.function - ); - } - selected.push(selected_id); - } - Ok(TriggerBindingV1 { - trigger_type: binding.trigger.as_str().to_string(), - function_id: function_id.clone(), - config: json!({ - "functions": selected, - "priority": binding.priority - }), - }) - }) - .collect() -} - -pub(super) fn function_call_ids( - authored: &AuthoredScenarioV1, -) -> anyhow::Result> { - let mut calls = Vec::new(); - let mut seen = BTreeSet::new(); - for (generation_index, generation) in authored.router.generations.iter().enumerate() { - match &generation.reply { - RouterReplyV1::FunctionCall { - id, - function, - arguments, - .. - } => { - let call_ordinal = calls.len() + 1; - let id = id.clone().unwrap_or_else(|| format!("call-{call_ordinal}")); - validate_function_arguments( - authored, - function, - arguments, - &format!("generation {}", generation_index + 1), - )?; - register_call( - &mut calls, - &mut seen, - CompiledFunctionCall { - id, - function: function.clone(), - generation_index, - }, - )?; - } - RouterReplyV1::Text { .. } => {} - } - } - Ok(calls) -} - -fn register_call( - calls: &mut Vec, - seen: &mut BTreeSet, - call: CompiledFunctionCall, -) -> anyhow::Result<()> { - if call.id.trim().is_empty() { - anyhow::bail!("function call id must not be empty"); - } - if !seen.insert(call.id.clone()) { - anyhow::bail!("duplicate function call id {:?}", call.id); - } - calls.push(call); - Ok(()) -} - -pub(super) fn compile_fault( - authored: &AuthoredScenarioV1, - function_ids: &BTreeMap, - calls: &[CompiledFunctionCall], -) -> anyhow::Result> { - let Some(fault) = &authored.fault else { - return Ok(None); - }; - if fault.after_target_calls == 0 { - anyhow::bail!("fault.after_target_calls must be greater than zero"); - } - let function = calls - .first() - .map(|call| call.function.as_str()) - .context("fault injection requires an authored function call")?; - let function_id = function_ids - .get(function) - .with_context(|| format!("fault references unknown function alias {function:?}"))?; - let matching_calls = calls - .iter() - .filter(|call| call.function == function) - .count() as u64; - if matching_calls < fault.after_target_calls { - anyhow::bail!( - "fault waits for {} call(s) to {function:?}, but only {matching_calls} are authored", - fault.after_target_calls - ); - } - Ok(Some(CompiledFaultV1 { - kind: fault.kind, - function_id: function_id.clone(), - after_target_calls: fault.after_target_calls, - restart_delay_ms: fault.restart_delay_ms, - })) -} - -pub(super) fn hold_fault_target( - recorder: &mut RecorderConfigV1, - function_id: &str, - call_ordinal: u64, -) -> anyhow::Result<()> { - let target = std::iter::once(&mut recorder.target) - .chain(recorder.extra_functions.iter_mut()) - .find(|target| target.function_id == function_id) - .with_context(|| format!("fault target {function_id:?} is not a controlled function"))?; - target.hold_response_at = Some(call_ordinal); - Ok(()) -} - -pub(super) fn compile_send( - authored: &AuthoredScenarioV1, - model: &ModelFixtureV1, - allowed_ids: &[String], -) -> anyhow::Result { - use crate::types::scenario::{ - CompiledFunctionExposureV1, CompiledFunctionPolicyV1, CompiledSendOptionsV1, CompiledSendV1, - }; - - Ok(CompiledSendV1 { - session_id: "{{session_id}}".to_string(), - message: authored.send.message.clone(), - model: model.id.clone(), - provider: model.provider.clone(), - idempotency_key: format!("{{{{run_id}}}}:{}", authored.id.to_ascii_lowercase()), - options: CompiledSendOptionsV1 { - functions: CompiledFunctionPolicyV1 { - allow: allowed_ids.to_vec(), - deny: Vec::new(), - expose: CompiledFunctionExposureV1::Native, - }, - }, - }) -} diff --git a/harness/evals/integration/src/expand/render.rs b/harness/evals/integration/src/expand/render.rs deleted file mode 100644 index 252a5079e..000000000 --- a/harness/evals/integration/src/expand/render.rs +++ /dev/null @@ -1,25 +0,0 @@ -use anyhow::Context; -use serde_json::json; - -use super::{expand_compiled_fixture, CompiledFixtureV1}; - -pub(super) fn validate_placeholders(fixture: &CompiledFixtureV1) -> anyhow::Result<()> { - render_compiled(fixture) - .map(|_| ()) - .context("validating compiled placeholders") -} - -/// Deterministic, fully expanded representation used by the `render` CLI. -pub fn render_compiled(fixture: &CompiledFixtureV1) -> anyhow::Result { - let run_id = format!("render-{}", fixture.scenario.id.to_ascii_lowercase()); - let session_id = format!( - "render-session-{}", - fixture.scenario.id.to_ascii_lowercase() - ); - let expanded = expand_compiled_fixture(fixture, &run_id, &session_id)?; - Ok(crate::canonical::canonical_json_pretty(&json!({ - "scenario": expanded.scenario, - "router_script": expanded.script, - "system_prompt": expanded.system_prompt - }))) -} diff --git a/harness/evals/integration/src/expand/router.rs b/harness/evals/integration/src/expand/router.rs deleted file mode 100644 index 7ab1f7f89..000000000 --- a/harness/evals/integration/src/expand/router.rs +++ /dev/null @@ -1,352 +0,0 @@ -use std::collections::BTreeMap; - -use anyhow::Context; -use serde_json::{json, Value}; - -use crate::types::frames::{ - AssistantMessage, AssistantMessageEvent, AssistantRoleTag, ContentBlock, RouterChatResponse, - StopReason, -}; -use crate::types::scenario::{AuthoredScenarioV1, RouterReplyV1}; -use crate::types::script::{ - GenerationMatchV1, JsonMatcherV1, JsonNormalizerV1, ModelFixtureV1, NormalizerOperation, - RouterScriptV1, SchemaVersion1, ScriptedGenerationV1, -}; - -use super::{validation::validate_reply, CompiledFunctionCall}; - -struct CompiledReply { - frames: Vec, - response: RouterChatResponse, - history: Vec, -} - -pub(super) fn compile_router( - authored: &AuthoredScenarioV1, - model: &ModelFixtureV1, - tools: &Value, - function_ids: &BTreeMap, - calls: &[CompiledFunctionCall], -) -> anyhow::Result { - let mut messages = vec![json!({ - "role": "user", - "content": [{ "type": "text", "text": authored.send.message }] - })]; - let mut generations = Vec::with_capacity(authored.router.generations.len()); - - for (index, authored_generation) in authored.router.generations.iter().enumerate() { - let ordinal = (index + 1) as u64; - let call_id = if matches!( - &authored_generation.reply, - RouterReplyV1::FunctionCall { .. } - ) { - let id = calls - .iter() - .find(|call| call.generation_index == index) - .context("compiler lost a function-call id")?; - Some(id.id.as_str()) - } else { - None - }; - validate_reply(&authored_generation.reply, ordinal, authored, function_ids)?; - let CompiledReply { - frames, - response, - history, - } = compile_reply( - &authored_generation.reply, - ordinal, - model, - authored, - function_ids, - call_id, - )?; - let mut match_ = default_match(ordinal, model, &messages, tools); - apply_match_overrides(&mut match_, &authored_generation.match_overrides); - generations.push(ScriptedGenerationV1 { - ordinal, - match_, - frames, - response, - }); - messages.extend(history); - } - - Ok(RouterScriptV1 { - schema_version: SchemaVersion1::V1, - scenario_id: authored.id.clone(), - model: model.clone(), - generations, - }) -} - -fn default_match( - ordinal: u64, - model: &ModelFixtureV1, - messages: &[Value], - tools: &Value, -) -> GenerationMatchV1 { - let normalize = (0..messages.len()) - .map(|index| JsonNormalizerV1 { - pointer: format!("/{index}/timestamp"), - operation: NormalizerOperation::Delete, - }) - .collect(); - GenerationMatchV1 { - writer_ref: JsonMatcherV1::Subset { - expected: json!({ "direction": "write" }), - normalize: None, - }, - request_id: JsonMatcherV1::Regex { - pattern: if ordinal == 1 { - "^t_[0-9a-f]{32}:[0-9]+$".to_string() - } else { - format!("^t_[0-9a-f]{{32}}:{}$", ordinal - 1) - }, - }, - model: JsonMatcherV1::Exact { - expected: json!(model.id), - normalize: None, - }, - provider: JsonMatcherV1::Exact { - expected: json!(model.provider), - normalize: None, - }, - system_prompt: JsonMatcherV1::Sha256 { - expected: "{{system_prompt_sha256}}".to_string(), - }, - messages: JsonMatcherV1::Exact { - expected: Value::Array(messages.to_vec()), - normalize: Some(normalize), - }, - tools: JsonMatcherV1::Exact { - expected: tools.clone(), - normalize: None, - }, - response_format: JsonMatcherV1::Absent, - thinking_level: JsonMatcherV1::Absent, - max_output_tokens: JsonMatcherV1::Absent, - provider_options: JsonMatcherV1::Absent, - metadata: JsonMatcherV1::Absent, - } -} - -fn apply_match_overrides( - target: &mut GenerationMatchV1, - overrides: &crate::types::scenario::GenerationMatchOverridesV1, -) { - macro_rules! replace { - ($field:ident) => { - if let Some(value) = &overrides.$field { - target.$field = value.clone(); - } - }; - } - replace!(request_id); - replace!(system_prompt); - replace!(messages); - replace!(tools); -} - -fn compile_reply( - reply: &RouterReplyV1, - ordinal: u64, - model: &ModelFixtureV1, - authored: &AuthoredScenarioV1, - function_ids: &BTreeMap, - call_id: Option<&str>, -) -> anyhow::Result { - match reply { - RouterReplyV1::Text { - text, - chunks, - usage, - } => { - let message = assistant_message( - vec![ContentBlock::Text { text: text.clone() }], - StopReason::End, - usage.clone(), - model, - ordinal as i64, - ); - let mut frames = Vec::new(); - if chunks.is_empty() { - frames.push(AssistantMessageEvent::Done { - message: message.clone(), - }); - } else { - frames.push(AssistantMessageEvent::Start { - partial: assistant_message( - Vec::new(), - StopReason::End, - None, - model, - ordinal as i64, - ), - }); - frames.push(AssistantMessageEvent::TextStart { - partial: assistant_message( - vec![ContentBlock::Text { - text: String::new(), - }], - StopReason::End, - None, - model, - ordinal as i64, - ), - }); - frames.extend(chunks.iter().cloned().map(|delta| { - AssistantMessageEvent::TextDelta { - partial: None, - delta, - } - })); - frames.push(AssistantMessageEvent::TextEnd { - partial: assistant_message( - vec![ContentBlock::Text { text: text.clone() }], - StopReason::End, - None, - model, - ordinal as i64, - ), - }); - if let Some(usage) = usage { - frames.push(AssistantMessageEvent::Usage { - usage: usage.clone(), - }); - } - frames.push(AssistantMessageEvent::Stop { - stop_reason: StopReason::End, - error_message: None, - error_kind: None, - }); - frames.push(AssistantMessageEvent::Done { - message: message.clone(), - }); - } - Ok(CompiledReply { - frames, - response: RouterChatResponse { - ok: true, - provider: model.provider.clone(), - model: model.id.clone(), - stop_reason: Some(StopReason::End), - usage: usage.clone(), - error: None, - }, - history: vec![json!({ - "role": "assistant", - "content": [{ "type": "text", "text": text }], - "stop_reason": "end", - "model": model.id, - "provider": model.provider - })], - }) - } - RouterReplyV1::FunctionCall { - function, - arguments, - usage, - .. - } => { - let call_id = call_id.context("missing compiled function-call id")?; - let message = assistant_message( - vec![ContentBlock::FunctionCall { - id: call_id.to_string(), - function_id: function_ids[function].clone(), - arguments: arguments.clone(), - }], - StopReason::FunctionCall, - usage.clone(), - model, - ordinal as i64, - ); - let function_id = &function_ids[function]; - let function_response = &authored.functions[function].response; - let (content, is_error) = normalize_function_response(function_response); - Ok(CompiledReply { - frames: vec![AssistantMessageEvent::Done { message }], - response: RouterChatResponse { - ok: true, - provider: model.provider.clone(), - model: model.id.clone(), - stop_reason: Some(StopReason::FunctionCall), - usage: usage.clone(), - error: None, - }, - history: vec![ - json!({ - "role": "assistant", - "content": [{ - "type": "function_call", - "id": call_id, - "function_id": function_id, - "arguments": arguments - }], - // The harness persists an open call with its default - // stop reason; the wire response remains - // `function_call`. - "stop_reason": "end", - "model": model.id, - "provider": model.provider - }), - json!({ - "role": "function_result", - "function_call_id": call_id, - "function_id": function_id, - "content": content, - "details": function_response, - "is_error": is_error - }), - ], - }) - } - } -} - -fn assistant_message( - content: Vec, - stop_reason: StopReason, - usage: Option, - model: &ModelFixtureV1, - timestamp: i64, -) -> AssistantMessage { - AssistantMessage { - role: AssistantRoleTag::Assistant, - content, - stop_reason, - native_stop_reason: None, - error_message: None, - error_kind: None, - warnings: None, - usage, - model: model.id.clone(), - provider: model.provider.clone(), - timestamp, - } -} - -/// Mirrors `harness::trigger::normalize`: the next router generation must -/// match the exact function-result shape the real harness persists. -fn normalize_function_response(response: &Value) -> (Value, bool) { - let is_error = response - .get("is_error") - .and_then(Value::as_bool) - .unwrap_or(false); - if let Value::String(text) = response { - return (json!([{ "type": "text", "text": text }]), is_error); - } - if let Some(content) = response.get("content") { - if let Ok(blocks) = serde_json::from_value::>(content.clone()) { - if !blocks.is_empty() { - return ( - serde_json::to_value(blocks).expect("content blocks serialize"), - is_error, - ); - } - } - } - let rendered = - serde_json::to_string(response).unwrap_or_else(|_| "".to_string()); - (json!([{ "type": "text", "text": rendered }]), is_error) -} diff --git a/harness/evals/integration/src/expand/templates.rs b/harness/evals/integration/src/expand/templates.rs deleted file mode 100644 index 47539f694..000000000 --- a/harness/evals/integration/src/expand/templates.rs +++ /dev/null @@ -1,147 +0,0 @@ -use std::collections::BTreeMap; - -use serde_json::json; - -use crate::types::scenario::{AuthoredScenarioV1, RouterReplyV1, ScenarioFunctionV1}; -use crate::types::script::JsonMatcherV1; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ScenarioTemplateKind { - Text, - Function, - Hook, - Crash, -} - -/// Minimal valid authored scenario used as a compact factory by unit and -/// contract tests. New checked-in scenarios are builder modules under -/// `src/scenarios`, not templates. -pub fn scenario_template( - id: &str, - description: &str, - kind: ScenarioTemplateKind, -) -> AuthoredScenarioV1 { - let mut functions = BTreeMap::new(); - let record = ScenarioFunctionV1 { - description: "Record one integration fixture value.".to_string(), - request_schema: json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }) - .as_object() - .cloned() - .expect("object"), - response: json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - expose: true, - }; - if kind != ScenarioTemplateKind::Text { - functions.insert("record".to_string(), record); - } - - let mut bindings = Vec::new(); - let mut release = None; - let mut fault = None; - let mut timeouts = crate::types::scenario::DeadlinesV1::default(); - if kind == ScenarioTemplateKind::Hook { - functions.insert( - "hook".to_string(), - ScenarioFunctionV1 { - description: "Hold the controlled call for explicit release.".to_string(), - request_schema: json!({ "type": "object" }) - .as_object() - .cloned() - .expect("object"), - response: json!({ "decision": "hold" }), - expose: false, - }, - ); - bindings.push(crate::types::scenario::TriggerBindingSpecV1 { - trigger: crate::types::scenario::TriggerKindV1::HookPreTrigger, - function: "hook".to_string(), - functions: vec!["record".to_string()], - priority: 10, - }); - release = Some(crate::types::scenario::ReleaseV1 { - function_call_id: "call-1".to_string(), - action: crate::types::scenario::ReleaseActionV1::Execute, - }); - } - if kind == ScenarioTemplateKind::Crash { - fault = Some(crate::types::scenario::FaultV1 { - kind: crate::types::scenario::FaultKind::EngineSigkill, - after_target_calls: 1, - restart_delay_ms: 1_500, - }); - timeouts.scenario_ms = 120_000; - } - - let first_reply = match kind { - ScenarioTemplateKind::Text => RouterReplyV1::Text { - text: "fixture complete".to_string(), - chunks: vec!["fixture ".to_string(), "complete".to_string()], - usage: None, - }, - _ => RouterReplyV1::FunctionCall { - id: None, - function: "record".to_string(), - arguments: json!({ "value": "expected" }), - usage: None, - }, - }; - let mut generations = vec![crate::types::scenario::ScenarioGenerationV1 { - reply: first_reply, - match_overrides: Default::default(), - }]; - if kind != ScenarioTemplateKind::Text { - let match_overrides = if matches!( - kind, - ScenarioTemplateKind::Hook | ScenarioTemplateKind::Crash - ) { - crate::types::scenario::GenerationMatchOverridesV1 { - request_id: Some(JsonMatcherV1::Regex { - pattern: "^t_[0-9a-f]{32}:[0-9]+$".to_string(), - }), - system_prompt: Some(JsonMatcherV1::Present), - messages: Some(JsonMatcherV1::Present), - tools: Some(JsonMatcherV1::Present), - } - } else { - Default::default() - }; - generations.push(crate::types::scenario::ScenarioGenerationV1 { - reply: RouterReplyV1::Text { - text: "recorded once".to_string(), - chunks: Vec::new(), - usage: None, - }, - match_overrides, - }); - } - - AuthoredScenarioV1 { - id: id.to_string(), - description: description.to_string(), - quarantine: false, - send: crate::types::scenario::ScenarioSendV1 { - message: if kind == ScenarioTemplateKind::Text { - "Return the fixture phrase.".to_string() - } else { - "Call the recorder once.".to_string() - }, - }, - functions, - router: crate::types::scenario::ScenarioRouterV1 { - model: None, - generations, - }, - bindings, - release, - fault, - timeouts, - } -} diff --git a/harness/evals/integration/src/expand/tests.rs b/harness/evals/integration/src/expand/tests.rs deleted file mode 100644 index 606f330b4..000000000 --- a/harness/evals/integration/src/expand/tests.rs +++ /dev/null @@ -1,312 +0,0 @@ -use std::collections::BTreeMap; - -use serde_json::json; - -use crate::types::scenario::{ - AuthoredScenarioV1, RouterReplyV1, ScenarioGenerationV1, ScenarioRouterV1, ScenarioSendV1, -}; -use crate::types::script::JsonMatcherV1; - -use super::{ - compile_scenario, render_compiled, scenario_template, Placeholders, ScenarioTemplateKind, -}; - -fn minimal(reply: RouterReplyV1) -> AuthoredScenarioV1 { - AuthoredScenarioV1 { - id: "E2E-T".to_string(), - description: "A focused compiler fixture.".to_string(), - quarantine: false, - send: ScenarioSendV1 { - message: "hello".to_string(), - }, - functions: BTreeMap::new(), - router: ScenarioRouterV1 { - model: None, - generations: vec![ScenarioGenerationV1 { - reply, - match_overrides: Default::default(), - }], - }, - bindings: Vec::new(), - release: None, - fault: None, - timeouts: Default::default(), - } -} - -#[test] -fn text_reply_compiles_stream_and_common_defaults() { - let authored = minimal(RouterReplyV1::Text { - text: "hello".to_string(), - chunks: vec!["hel".to_string(), "lo".to_string()], - usage: None, - }); - let compiled = compile_scenario(&authored, "base\n").unwrap(); - assert_eq!(compiled.script.generations[0].frames.len(), 7); - assert!(compiled.scenario.send.options.functions.allow.is_empty()); - assert_eq!(compiled.scenario.deadlines.teardown_ms, 15_000); - assert!(compiled - .system_prompt_template - .ends_with("Function dispatch is entirely disabled this turn — do not call any function.")); -} - -#[test] -fn mismatched_chunks_and_unknown_aliases_fail_before_runtime() { - let authored = minimal(RouterReplyV1::Text { - text: "hello".to_string(), - chunks: vec!["wrong".to_string()], - usage: None, - }); - assert!( - format!("{:#}", compile_scenario(&authored, "base").unwrap_err()) - .contains("chunks concatenate") - ); - - let authored = minimal(RouterReplyV1::FunctionCall { - id: None, - function: "missing".to_string(), - arguments: json!({}), - usage: None, - }); - assert!( - format!("{:#}", compile_scenario(&authored, "base").unwrap_err()).contains("unknown alias") - ); -} - -#[test] -fn compile_rejects_unknown_placeholders_and_unsafe_ids() { - let mut authored = minimal(RouterReplyV1::Text { - text: "hello".to_string(), - chunks: Vec::new(), - usage: None, - }); - authored.send.message = "{{unknown}}".to_string(); - assert!( - format!("{:#}", compile_scenario(&authored, "base").unwrap_err()) - .contains("unexpanded placeholder") - ); - - authored.send.message = "hello".to_string(); - authored.id = "../../escape".to_string(); - assert!( - format!("{:#}", compile_scenario(&authored, "base").unwrap_err()).contains("scenario id") - ); -} - -#[test] -fn function_call_defaults_are_ordered_by_call_and_validated() { - let mut authored = scenario_template( - "E2E-CALLS", - "Validate generated function call ids.", - ScenarioTemplateKind::Function, - ); - authored.router.generations.insert( - 0, - ScenarioGenerationV1 { - reply: RouterReplyV1::Text { - text: "before call".to_string(), - chunks: Vec::new(), - usage: None, - }, - match_overrides: Default::default(), - }, - ); - let compiled = compile_scenario(&authored, "base").unwrap(); - let messages = match &compiled.script.generations[2].match_.messages { - JsonMatcherV1::Exact { expected, .. } => expected.as_array().unwrap(), - other => panic!("expected exact history, got {other:?}"), - }; - assert_eq!(messages[2]["content"][0]["id"], "call-1"); - assert_eq!(messages[3]["function_call_id"], "call-1"); - - let RouterReplyV1::FunctionCall { id, .. } = &mut authored.router.generations[1].reply else { - unreachable!() - }; - *id = Some(String::new()); - assert!( - format!("{:#}", compile_scenario(&authored, "base").unwrap_err()) - .contains("must not be empty") - ); -} - -#[test] -fn tools_are_canonicalized_and_function_contracts_validate() { - let mut authored = scenario_template( - "E2E-TOOLS", - "Validate tool ordering and arguments.", - ScenarioTemplateKind::Function, - ); - let record = authored.functions["record"].clone(); - authored - .functions - .insert("zeta".to_string(), record.clone()); - authored.functions.insert("alpha".to_string(), record); - let compiled = compile_scenario(&authored, "base").unwrap(); - let allowed = &compiled.scenario.send.options.functions.allow; - assert_eq!( - allowed, - &[ - "{{run_id}}::alpha".to_string(), - "{{run_id}}::record".to_string(), - "{{run_id}}::zeta".to_string() - ] - ); - let JsonMatcherV1::Exact { expected, .. } = &compiled.script.generations[0].match_.tools else { - panic!("tools must use an exact matcher"); - }; - assert_eq!( - expected - .as_array() - .unwrap() - .iter() - .map(|tool| tool["name"].as_str().unwrap()) - .collect::>(), - [ - "{{run_id}}::alpha", - "{{run_id}}::record", - "{{run_id}}::zeta" - ] - ); - - let RouterReplyV1::FunctionCall { arguments, .. } = &mut authored.router.generations[0].reply - else { - unreachable!() - }; - *arguments = json!({}); - assert!( - format!("{:#}", compile_scenario(&authored, "base").unwrap_err()) - .contains("do not match request_schema") - ); - - authored.functions.get_mut("record").unwrap().request_schema = - json!({ "type": "not-a-json-schema-type" }) - .as_object() - .unwrap() - .clone(); - assert!( - format!("{:#}", compile_scenario(&authored, "base").unwrap_err()) - .contains("invalid request_schema") - ); -} - -#[test] -fn function_response_history_matches_harness_normalization() { - let mut authored = scenario_template( - "E2E-RESPONSE", - "Validate normalized function results.", - ScenarioTemplateKind::Function, - ); - authored.functions.get_mut("record").unwrap().response = json!("ok"); - let compiled = compile_scenario(&authored, "base").unwrap(); - let JsonMatcherV1::Exact { expected, .. } = &compiled.script.generations[1].match_.messages - else { - panic!("function history must be exact"); - }; - assert_eq!( - expected[2]["content"], - json!([{ "type": "text", "text": "ok" }]) - ); - - authored.functions.get_mut("record").unwrap().response = json!({ "value": 1 }); - let compiled = compile_scenario(&authored, "base").unwrap(); - let JsonMatcherV1::Exact { expected, .. } = &compiled.script.generations[1].match_.messages - else { - unreachable!() - }; - assert_eq!( - expected[2]["content"], - json!([{ "type": "text", "text": "{\"value\":1}" }]) - ); -} - -#[test] -fn bindings_release_and_fault_fail_fast_when_incoherent() { - let mut hook = scenario_template( - "E2E-HOOK", - "Validate hook relationships.", - ScenarioTemplateKind::Hook, - ); - hook.bindings[0].functions.clear(); - assert!( - format!("{:#}", compile_scenario(&hook, "base").unwrap_err()) - .contains("select at least one") - ); - - hook.bindings[0].functions = vec!["record".to_string()]; - hook.functions.get_mut("hook").unwrap().response = json!({ "decision": "continue" }); - assert!( - format!("{:#}", compile_scenario(&hook, "base").unwrap_err()) - .contains("requires a selected hook with decision: hold") - ); - - let mut crash = scenario_template( - "E2E-CRASH", - "Validate fault relationships.", - ScenarioTemplateKind::Crash, - ); - let compiled = compile_scenario(&crash, "base").unwrap(); - assert_eq!( - compiled.scenario.recorder.target.hold_response_at, - Some(1), - "fault targets receive the compiler-owned response-gate ordinal" - ); - - let second_call = crash.router.generations[0].clone(); - crash.router.generations.insert(1, second_call); - crash.fault.as_mut().unwrap().after_target_calls = 2; - let compiled = compile_scenario(&crash, "base").unwrap(); - assert_eq!( - compiled.scenario.recorder.target.hold_response_at, - Some(2), - "earlier calls must return before the selected fault ordinal is held" - ); - - crash.fault.as_mut().unwrap().after_target_calls = 0; - assert!( - format!("{:#}", compile_scenario(&crash, "base").unwrap_err()) - .contains("after_target_calls must be greater than zero") - ); -} - -#[test] -fn templates_compile_and_render_deterministically() { - for kind in [ - ScenarioTemplateKind::Text, - ScenarioTemplateKind::Function, - ScenarioTemplateKind::Hook, - ScenarioTemplateKind::Crash, - ] { - let authored = scenario_template("E2E-NEW", "A generated scenario.", kind); - let fixture = compile_scenario(&authored, "base\n").unwrap(); - assert_eq!( - render_compiled(&fixture).unwrap(), - render_compiled(&fixture).unwrap() - ); - if matches!( - kind, - ScenarioTemplateKind::Hook | ScenarioTemplateKind::Crash - ) { - assert!(matches!( - fixture.script.generations[1].match_.messages, - JsonMatcherV1::Present - )); - } - } -} - -#[test] -fn expands_strings_keys_and_rejects_unknown_tokens() { - let placeholders = Placeholders::new("r1", "s_abc"); - let mut value = json!({ - "idempotency_key": "{{run_id}}:streamed-text", - "session_id": "{{session_id}}", - "{{run_id}}::record": { "count": 1 } - }); - placeholders.expand_value(&mut value).unwrap(); - assert_eq!(value["idempotency_key"], "r1:streamed-text"); - assert_eq!(value["session_id"], "s_abc"); - assert!(value.get("r1::record").is_some()); - - let mut bad = json!("{{unknown_token}}"); - assert!(placeholders.expand_value(&mut bad).is_err()); -} diff --git a/harness/evals/integration/src/expand/validation.rs b/harness/evals/integration/src/expand/validation.rs deleted file mode 100644 index 6a98db0c7..000000000 --- a/harness/evals/integration/src/expand/validation.rs +++ /dev/null @@ -1,162 +0,0 @@ -use std::collections::BTreeMap; - -use anyhow::Context; -use serde_json::Value; - -use crate::types::scenario::{validate_scenario_id, AuthoredScenarioV1, RouterReplyV1}; - -use super::CompiledFunctionCall; - -pub(super) fn validate_identity(authored: &AuthoredScenarioV1) -> anyhow::Result<()> { - validate_scenario_id(&authored.id)?; - if authored.description.trim().is_empty() { - anyhow::bail!("scenario description must not be empty"); - } - if authored.timeouts.readiness_ms == 0 - || authored.timeouts.scenario_ms == 0 - || authored.timeouts.teardown_ms == 0 - { - anyhow::bail!("readiness, scenario, and teardown timeouts must be greater than zero"); - } - for alias in authored.functions.keys() { - validate_alias(alias)?; - } - validate_function_schemas(authored)?; - Ok(()) -} - -fn validate_alias(alias: &str) -> anyhow::Result<()> { - if alias.is_empty() - || !alias - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) - { - anyhow::bail!( - "function alias {alias:?} must contain only ASCII letters, digits, '-' or '_'" - ); - } - Ok(()) -} - -fn validate_function_schemas(authored: &AuthoredScenarioV1) -> anyhow::Result<()> { - for (alias, function) in &authored.functions { - let schema = Value::Object(function.request_schema.clone()); - jsonschema::JSONSchema::compile(&schema).map_err(|error| { - anyhow::anyhow!("function {alias:?} has an invalid request_schema: {error}") - })?; - } - Ok(()) -} - -pub(super) fn validate_function_arguments( - authored: &AuthoredScenarioV1, - function: &str, - arguments: &Value, - context: &str, -) -> anyhow::Result<()> { - let controlled = authored.functions.get(function).with_context(|| { - format!("{context}: function call references unknown alias {function:?}") - })?; - let schema = Value::Object(controlled.request_schema.clone()); - let validator = jsonschema::JSONSchema::compile(&schema).map_err(|error| { - anyhow::anyhow!("{context}: function {function:?} has an invalid request_schema: {error}") - })?; - if let Err(errors) = validator.validate(arguments) { - let details = errors - .take(5) - .map(|error| error.to_string()) - .collect::>() - .join("; "); - anyhow::bail!( - "{context}: arguments for function {function:?} do not match request_schema: {details}" - ); - } - Ok(()) -} - -pub(super) fn validate_hook_response(alias: &str, response: &Value) -> anyhow::Result<()> { - let object = response - .as_object() - .with_context(|| format!("binding callback {alias:?} response must be an object"))?; - if let Some(decision) = object.get("decision") { - let decision = decision.as_str().with_context(|| { - format!("binding callback {alias:?} response decision must be a string") - })?; - if !matches!(decision, "continue" | "deny" | "hold") { - anyhow::bail!("binding callback {alias:?} has unsupported decision {decision:?}"); - } - } - if object - .get("mutations") - .is_some_and(|mutations| !mutations.is_object()) - { - anyhow::bail!("binding callback {alias:?} response mutations must be an object"); - } - Ok(()) -} - -pub(super) fn validate_release( - authored: &AuthoredScenarioV1, - calls: &[CompiledFunctionCall], -) -> anyhow::Result<()> { - let Some(release) = &authored.release else { - return Ok(()); - }; - let Some(call) = calls - .iter() - .find(|call| call.id == release.function_call_id) - else { - anyhow::bail!( - "release references unknown function call {:?}", - release.function_call_id - ); - }; - let held = authored.bindings.iter().any(|binding| { - binding.functions.contains(&call.function) - && authored.functions[&binding.function] - .response - .get("decision") - .and_then(Value::as_str) - == Some("hold") - }); - if !held { - anyhow::bail!( - "release for function call {:?} requires a selected hook with decision: hold", - release.function_call_id - ); - } - Ok(()) -} - -pub(super) fn validate_reply( - reply: &RouterReplyV1, - ordinal: u64, - authored: &AuthoredScenarioV1, - function_ids: &BTreeMap, -) -> anyhow::Result<()> { - match reply { - RouterReplyV1::Text { text, chunks, .. } => { - if !chunks.is_empty() && chunks.concat() != *text { - anyhow::bail!( - "generation {ordinal}: text chunks concatenate to {:?}, expected {:?}", - chunks.concat(), - text - ); - } - } - RouterReplyV1::FunctionCall { function, .. } => { - if !function_ids.contains_key(function) { - anyhow::bail!( - "generation {ordinal}: function call references unknown alias {function:?}" - ); - } - let allowed = authored.functions[function].expose; - if !allowed { - anyhow::bail!( - "generation {ordinal}: function call alias {function:?} is not exposed by send" - ); - } - } - } - Ok(()) -} diff --git a/harness/evals/integration/src/fixtures.rs b/harness/evals/integration/src/fixtures.rs index d614703bf..21469ec77 100644 --- a/harness/evals/integration/src/fixtures.rs +++ b/harness/evals/integration/src/fixtures.rs @@ -7,12 +7,9 @@ mod discovery; mod loading; -mod script_validation; -mod stream_validation; pub use discovery::scenario_fixtures; pub use loading::ScenarioFixture; -pub use script_validation::validate_script; #[cfg(test)] mod tests; diff --git a/harness/evals/integration/src/fixtures/discovery.rs b/harness/evals/integration/src/fixtures/discovery.rs index 05aa1dfee..ced4df9f0 100644 --- a/harness/evals/integration/src/fixtures/discovery.rs +++ b/harness/evals/integration/src/fixtures/discovery.rs @@ -1,82 +1,52 @@ use super::loading::ScenarioFixture; -use crate::scenarios::RegisteredScenario; -/// Resolve `--scenario ` against the code registry and return -/// already-compiled fixtures. -/// -/// `include_quarantined` is intended for validation. Explicit id/slug -/// selection always includes the requested fixture; for `all`, normal runs -/// exclude quarantines while validation includes them. -pub fn scenario_fixtures( - selector: &str, - include_quarantined: bool, -) -> anyhow::Result> { - select_fixtures(crate::scenarios::all(), selector, include_quarantined) +/// Resolve `--scenario ` against the checked-in fixtures. +pub fn scenario_fixtures(selector: &str) -> anyhow::Result> { + select_fixtures(crate::scenarios::all(), selector) } -/// Registry-parameterized core so selection semantics stay testable without -/// the checked-in scenario set. pub(crate) fn select_fixtures( - registered: Vec, + fixtures: Vec, selector: &str, - include_quarantined: bool, ) -> anyhow::Result> { - reject_duplicate_identities(®istered)?; + reject_duplicate_identities(&fixtures)?; if selector == "all" { - let mut selected = Vec::new(); - for entry in ®istered { - if include_quarantined || !entry.authored.quarantine { - selected.push(ScenarioFixture::from_registered(entry)?); - } + anyhow::ensure!(!fixtures.is_empty(), "selector \"all\" matched no scenario"); + for fixture in &fixtures { + fixture.validate()?; } - if selected.is_empty() { - let qualifier = if include_quarantined { - "" - } else { - " non-quarantined" - }; - anyhow::bail!("selector \"all\" matched no{qualifier} registered scenario"); - } - return Ok(selected); + return Ok(fixtures); } - // Slug and id selection compile only the requested scenario, so an - // unrelated fixture that fails compilation cannot block it. Malformed - // unrelated fixtures remain the responsibility of `validate --scenario - // all`. - if let Some(entry) = registered.iter().find(|entry| entry.slug == selector) { - return Ok(vec![ScenarioFixture::from_registered(entry)?]); - } - if let Some(entry) = registered - .iter() - .find(|entry| entry.authored.id == selector) + if let Some(fixture) = fixtures + .into_iter() + .find(|fixture| fixture.slug == selector || fixture.scenario.id == selector) { - return Ok(vec![ScenarioFixture::from_registered(entry)?]); + fixture.validate()?; + return Ok(vec![fixture]); } anyhow::bail!("no scenario matches selector {selector:?}") } -/// Identity checks read only authored data, never compile, so they cannot be -/// masked by an unrelated compilation failure. -fn reject_duplicate_identities(registered: &[RegisteredScenario]) -> anyhow::Result<()> { +fn reject_duplicate_identities(fixtures: &[ScenarioFixture]) -> anyhow::Result<()> { let mut slugs = std::collections::BTreeMap::new(); let mut ids = std::collections::BTreeMap::new(); - for entry in registered { - if let Some(previous) = slugs.insert(entry.slug.clone(), entry.authored.id.clone()) { + for fixture in fixtures { + if let Some(previous) = slugs.insert(fixture.slug.clone(), fixture.scenario.id.clone()) { anyhow::bail!( "duplicate scenario slug {:?} (ids {:?} and {:?})", - entry.slug, + fixture.slug, previous, - entry.authored.id + fixture.scenario.id ); } - if let Some(previous) = ids.insert(entry.authored.id.clone(), entry.slug.clone()) { + if let Some(previous) = ids.insert(fixture.scenario.id.clone(), fixture.slug.clone()) { anyhow::bail!( "duplicate scenario id {:?} in {:?} and {:?}", - entry.authored.id, + fixture.scenario.id, previous, - entry.slug + fixture.slug ); } } diff --git a/harness/evals/integration/src/fixtures/loading.rs b/harness/evals/integration/src/fixtures/loading.rs index 1b69982e2..57d109a66 100644 --- a/harness/evals/integration/src/fixtures/loading.rs +++ b/harness/evals/integration/src/fixtures/loading.rs @@ -1,18 +1,12 @@ -use anyhow::Context; - -use super::script_validation::validate_script; -use crate::expand::{compile_scenario, CompiledFixtureV1}; -use crate::scenarios::{RegisteredScenario, ScenarioDriver, VerifyFn}; +use crate::expand::CompiledFixtureV1; +use crate::scenarios::{ScenarioDriver, VerifyFn}; use crate::types::scenario::CompiledScenarioV1; use crate::types::script::RouterScriptV1; -const DEFAULT_SYSTEM_PROMPT: &str = include_str!("../../../../prompts/default.txt"); - #[derive(Debug, Clone)] pub struct ScenarioFixture { pub slug: String, pub driver: ScenarioDriver, - pub quarantine: bool, pub scenario: CompiledScenarioV1, pub script: RouterScriptV1, /// Compiled Harness default plus inferred session/policy aid. @@ -22,29 +16,31 @@ pub struct ScenarioFixture { } impl ScenarioFixture { - /// Compile one registered scenario against the Harness default prompt. - pub fn from_registered(entry: &RegisteredScenario) -> anyhow::Result { - let CompiledFixtureV1 { - scenario: compiled, - script, - system_prompt_template, - } = compile_scenario(&entry.authored, DEFAULT_SYSTEM_PROMPT) - .with_context(|| format!("compiling scenario {}", entry.slug))?; - - let fixture = ScenarioFixture { - slug: entry.slug.clone(), - driver: entry.driver, - quarantine: entry.authored.quarantine, - scenario: compiled, - script, - system_prompt_template, - verify: entry.verify, - }; - fixture.validate()?; - Ok(fixture) - } - pub fn validate(&self) -> anyhow::Result<()> { + anyhow::ensure!( + !self.slug.is_empty() + && self + .slug + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')), + "scenario slug {:?} is not filesystem-safe", + self.slug + ); + anyhow::ensure!( + !self.scenario.id.is_empty() + && self.scenario.id.len() <= 128 + && self + .scenario + .id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')), + "scenario id {:?} is not filesystem-safe", + self.scenario.id + ); + anyhow::ensure!( + !self.scenario.description.trim().is_empty(), + "scenario description must not be empty" + ); if self.script.scenario_id != self.scenario.id { anyhow::bail!( "script scenario_id {:?} does not match scenario id {:?}", @@ -52,26 +48,50 @@ impl ScenarioFixture { self.scenario.id ); } - for declared in std::iter::once(&self.scenario.recorder.target) - .chain(self.scenario.recorder.extra_functions.iter()) + if !self + .scenario + .recorder + .target + .function_id + .starts_with("{{run_id}}::") { - if !declared.function_id.starts_with("{{run_id}}::") { - anyhow::bail!( - "recorder function {:?} must be scoped by the {{{{run_id}}}}:: prefix", - declared.function_id - ); - } + anyhow::bail!( + "recorder function {:?} must be run-scoped", + self.scenario.recorder.target.function_id + ); } - for binding in &self.scenario.bindings { - if !binding.function_id.starts_with("{{run_id}}::") { - anyhow::bail!( - "scenario binding {:?} must bind a run-scoped function", - binding.function_id - ); - } + anyhow::ensure!( + !self.script.generations.is_empty(), + "router script has no generations" + ); + let mut ordinals = std::collections::BTreeSet::new(); + for generation in &self.script.generations { + anyhow::ensure!( + ordinals.insert(generation.ordinal), + "duplicate router generation ordinal {}", + generation.ordinal + ); + anyhow::ensure!( + generation + .frames + .last() + .is_some_and(|frame| frame.is_terminal()), + "generation {} does not end in a terminal frame", + generation.ordinal + ); + anyhow::ensure!( + !generation.frames[..generation.frames.len() - 1] + .iter() + .any(|frame| frame.is_terminal()), + "generation {} contains an early terminal frame", + generation.ordinal + ); } - validate_script(&self.script) - .with_context(|| format!("router script for {}", self.scenario.id))?; + crate::expand::expand_compiled_fixture( + &self.compiled(), + "validate-run", + "validate-session", + )?; Ok(()) } diff --git a/harness/evals/integration/src/fixtures/script_validation.rs b/harness/evals/integration/src/fixtures/script_validation.rs deleted file mode 100644 index e2fa49c8d..000000000 --- a/harness/evals/integration/src/fixtures/script_validation.rs +++ /dev/null @@ -1,248 +0,0 @@ -use anyhow::Context; - -use crate::types::script::{JsonMatcherV1, JsonNormalizerV1, NormalizerOperation, RouterScriptV1}; - -pub fn validate_script(script: &RouterScriptV1) -> anyhow::Result<()> { - if script.generations.is_empty() { - anyhow::bail!("script has no generations"); - } - let mut seen = std::collections::BTreeSet::new(); - for generation in &script.generations { - if !seen.insert(generation.ordinal) { - anyhow::bail!("duplicate generation ordinal {}", generation.ordinal); - } - for (field, matcher) in generation.match_.fields() { - validate_matcher(matcher) - .with_context(|| format!("generation {} field {field}", generation.ordinal))?; - } - let terminal_positions: Vec = generation - .frames - .iter() - .enumerate() - .filter(|(_, frame)| frame.is_terminal()) - .map(|(index, _)| index) - .collect(); - match terminal_positions.as_slice() { - [] => anyhow::bail!("generation {} has no terminal frame", generation.ordinal), - [last] if *last == generation.frames.len() - 1 => {} - [_] => anyhow::bail!( - "generation {}: terminal frame is not the last frame", - generation.ordinal - ), - _ => anyhow::bail!( - "generation {} has multiple terminal frames", - generation.ordinal - ), - } - validate_response_agreement(generation)?; - } - Ok(()) -} - -fn validate_matcher(matcher: &JsonMatcherV1) -> anyhow::Result<()> { - match matcher { - JsonMatcherV1::Regex { pattern } => { - regex::Regex::new(pattern) - .map_err(|error| anyhow::anyhow!("invalid regex {pattern:?}: {error}"))?; - } - JsonMatcherV1::Sha256 { expected } => { - let is_placeholder = expected.contains("{{"); - let is_hex = - expected.len() == 64 && expected.bytes().all(|byte| byte.is_ascii_hexdigit()); - if !is_placeholder && !is_hex { - anyhow::bail!("sha256 expected value must be 64 hex chars, got {expected:?}"); - } - } - JsonMatcherV1::Exact { normalize, .. } | JsonMatcherV1::Subset { normalize, .. } => { - for normalizer in normalize.as_deref().unwrap_or_default() { - validate_normalizer(normalizer)?; - } - } - JsonMatcherV1::Absent | JsonMatcherV1::Present => {} - } - Ok(()) -} - -fn validate_normalizer(normalizer: &JsonNormalizerV1) -> anyhow::Result<()> { - crate::matcher::validate_pointer(&normalizer.pointer)?; - match normalizer.operation { - NormalizerOperation::Delete if normalizer.pointer.is_empty() => { - anyhow::bail!("delete normalizer cannot target the document root") - } - NormalizerOperation::Delete => Ok(()), - } -} - -/// Terminal frame and scripted response must agree (spec: "response/frame -/// disagreement" is rejected at load): `done` requires `ok:true` and a -/// matching `stop_reason`; `error` requires `ok:false` with an error shape. -fn validate_response_agreement( - generation: &crate::types::script::ScriptedGenerationV1, -) -> anyhow::Result<()> { - use crate::types::frames::AssistantMessageEvent as Frame; - - let terminal = generation.frames.last().expect("validated non-empty"); - match terminal { - Frame::Done { message } => { - if !generation.response.ok { - anyhow::bail!( - "generation {}: done frame with ok:false response", - generation.ordinal - ); - } - if generation.response.error.is_some() { - anyhow::bail!( - "generation {}: successful response must not carry an error", - generation.ordinal - ); - } - validate_terminal_message(generation, message)?; - validate_stream_agreement(generation, message)?; - } - Frame::Error { error } => { - if generation.response.ok || generation.response.error.is_none() { - anyhow::bail!( - "generation {}: error frame requires ok:false and an error shape", - generation.ordinal - ); - } - validate_terminal_message(generation, error)?; - let response_error = generation.response.error.as_ref().expect("checked"); - let message_error = error.error_message.as_deref(); - if message_error != Some(response_error.message.as_str()) { - anyhow::bail!( - "generation {}: response error message disagrees with terminal error frame", - generation.ordinal - ); - } - let message_code = error.error_kind.map(|kind| { - serde_json::to_value(kind) - .expect("error kind serializes") - .as_str() - .expect("error kind is a string") - .to_string() - }); - if message_code.as_deref() != Some(response_error.code.as_str()) { - anyhow::bail!( - "generation {}: response error code disagrees with terminal error frame", - generation.ordinal - ); - } - validate_stream_agreement(generation, error)?; - } - _ => unreachable!("terminal position validated"), - } - Ok(()) -} - -fn validate_terminal_message( - generation: &crate::types::script::ScriptedGenerationV1, - message: &crate::types::frames::AssistantMessage, -) -> anyhow::Result<()> { - if generation.response.provider != message.provider { - anyhow::bail!( - "generation {}: response provider disagrees with terminal message", - generation.ordinal - ); - } - if generation.response.model != message.model { - anyhow::bail!( - "generation {}: response model disagrees with terminal message", - generation.ordinal - ); - } - if generation.response.stop_reason != Some(message.stop_reason) { - anyhow::bail!( - "generation {}: response stop_reason disagrees with terminal message", - generation.ordinal - ); - } - if generation.response.usage != message.usage { - anyhow::bail!( - "generation {}: response usage disagrees with terminal message", - generation.ordinal - ); - } - Ok(()) -} - -fn validate_stream_agreement( - generation: &crate::types::script::ScriptedGenerationV1, - terminal: &crate::types::frames::AssistantMessage, -) -> anyhow::Result<()> { - use crate::types::frames::AssistantMessageEvent as Frame; - - let mut streamed_usage = None; - let mut streamed_stop = None; - - for frame in &generation.frames[..generation.frames.len() - 1] { - let partial = match frame { - Frame::Start { partial } - | Frame::TextStart { partial } - | Frame::TextEnd { partial } - | Frame::ThinkingStart { partial } - | Frame::ThinkingEnd { partial } - | Frame::FunctioncallStart { partial } - | Frame::FunctioncallEnd { partial } => Some(partial), - Frame::TextDelta { - partial: Some(partial), - .. - } - | Frame::ThinkingDelta { - partial: Some(partial), - .. - } - | Frame::FunctioncallDelta { - partial: Some(partial), - .. - } => Some(partial), - _ => None, - }; - if let Some(partial) = partial { - if partial.provider != terminal.provider || partial.model != terminal.model { - anyhow::bail!( - "generation {}: streamed partial provider/model disagrees with terminal message", - generation.ordinal - ); - } - } - - match frame { - Frame::Usage { usage } => streamed_usage = Some(usage.clone()), - Frame::Stop { - stop_reason, - error_message, - error_kind, - } => { - streamed_stop = Some((*stop_reason, error_message.clone(), *error_kind)); - } - _ => {} - } - } - - super::stream_validation::validate_stream_content( - &generation.frames[..generation.frames.len() - 1], - terminal, - generation.ordinal, - )?; - if let Some(usage) = streamed_usage { - if terminal.usage.as_ref() != Some(&usage) { - anyhow::bail!( - "generation {}: streamed usage disagrees with terminal message", - generation.ordinal - ); - } - } - if let Some((stop_reason, error_message, error_kind)) = streamed_stop { - if stop_reason != terminal.stop_reason - || error_message != terminal.error_message - || error_kind != terminal.error_kind - { - anyhow::bail!( - "generation {}: stop frame disagrees with terminal message", - generation.ordinal - ); - } - } - Ok(()) -} diff --git a/harness/evals/integration/src/fixtures/stream_validation.rs b/harness/evals/integration/src/fixtures/stream_validation.rs deleted file mode 100644 index 5b6ae3cb9..000000000 --- a/harness/evals/integration/src/fixtures/stream_validation.rs +++ /dev/null @@ -1,343 +0,0 @@ -//! Content-level agreement between streamed frames and the terminal message. -//! -//! Boundary snapshots are authoritative for reconstruction, but they must not -//! hide contradictory deltas that consumers already observed. Validate each -//! open block before accepting its end snapshot, then compare the fully -//! reconstructed content with the terminal frame. - -use crate::types::frames::{AssistantMessage, AssistantMessageEvent as Frame, ContentBlock}; - -#[derive(Default)] -struct StreamState { - base: Option, - open: Option, - saw_content: bool, -} - -enum OpenBlock { - Text { seed: String, deltas: String }, - Thinking { seed: String, deltas: String }, - FunctionCall { id: String, arguments: String }, -} - -pub(super) fn validate_stream_content( - frames: &[Frame], - terminal: &AssistantMessage, - ordinal: u64, -) -> anyhow::Result<()> { - let mut state = StreamState::default(); - for frame in frames { - state.apply(frame, ordinal)?; - } - if !state.saw_content { - return Ok(()); - } - - let reconstructed = state.reconstructed_content(ordinal)?; - if !content_equal_allowing_pending_thinking_signature(&reconstructed, &terminal.content) { - anyhow::bail!("generation {ordinal}: streamed content disagrees with terminal message"); - } - Ok(()) -} - -impl StreamState { - fn apply(&mut self, frame: &Frame, ordinal: u64) -> anyhow::Result<()> { - match frame { - Frame::Start { partial } => { - self.base = Some(partial.clone()); - self.open = None; - self.saw_content |= !partial.content.is_empty(); - } - Frame::TextStart { partial } => { - self.start_text(partial); - } - Frame::TextDelta { - partial: Some(partial), - .. - } => { - self.base = Some(partial.clone()); - self.open = None; - self.saw_content = true; - } - Frame::TextDelta { - partial: None, - delta, - } => { - self.saw_content = true; - match &mut self.open { - Some(OpenBlock::Text { deltas, .. }) => deltas.push_str(delta), - Some(_) => anyhow::bail!( - "generation {ordinal}: text delta arrived while another block was open" - ), - None => { - self.open = Some(OpenBlock::Text { - seed: String::new(), - deltas: delta.clone(), - }); - } - } - } - Frame::TextEnd { partial } => { - self.validate_boundary(partial, BoundaryKind::Text, ordinal)?; - self.base = Some(partial.clone()); - self.open = None; - self.saw_content = true; - } - Frame::ThinkingStart { partial } => { - self.start_thinking(partial); - } - Frame::ThinkingDelta { - partial: Some(partial), - .. - } => { - self.base = Some(partial.clone()); - self.open = None; - self.saw_content = true; - } - Frame::ThinkingDelta { - partial: None, - delta, - } => { - self.saw_content = true; - match &mut self.open { - Some(OpenBlock::Thinking { deltas, .. }) => deltas.push_str(delta), - Some(_) => anyhow::bail!( - "generation {ordinal}: thinking delta arrived while another block was open" - ), - None => { - self.open = Some(OpenBlock::Thinking { - seed: String::new(), - deltas: delta.clone(), - }); - } - } - } - Frame::ThinkingEnd { partial } => { - self.validate_boundary(partial, BoundaryKind::Thinking, ordinal)?; - self.base = Some(partial.clone()); - self.open = None; - self.saw_content = true; - } - Frame::FunctioncallStart { partial } => { - self.base = Some(partial.clone()); - self.open = Some(OpenBlock::FunctionCall { - id: String::new(), - arguments: String::new(), - }); - self.saw_content = true; - } - Frame::FunctioncallDelta { - partial: Some(partial), - .. - } => { - self.base = Some(partial.clone()); - self.open = None; - self.saw_content = true; - } - Frame::FunctioncallDelta { - partial: None, - delta, - id, - } => { - self.saw_content = true; - match &mut self.open { - Some(OpenBlock::FunctionCall { - id: open_id, - arguments, - }) => { - if !id.is_empty() { - if !open_id.is_empty() && open_id != id { - anyhow::bail!( - "generation {ordinal}: function-call delta changed call id" - ); - } - *open_id = id.clone(); - } - arguments.push_str(delta); - } - Some(_) => anyhow::bail!( - "generation {ordinal}: function-call delta arrived while another block was open" - ), - None => { - self.open = Some(OpenBlock::FunctionCall { - id: id.clone(), - arguments: delta.clone(), - }); - } - } - } - Frame::FunctioncallEnd { partial } => { - self.validate_boundary(partial, BoundaryKind::FunctionCall, ordinal)?; - self.base = Some(partial.clone()); - self.open = None; - self.saw_content = true; - } - Frame::Usage { .. } - | Frame::Ping - | Frame::Stop { .. } - | Frame::Done { .. } - | Frame::Error { .. } => {} - } - Ok(()) - } - - fn start_text(&mut self, partial: &AssistantMessage) { - let mut base = partial.clone(); - let seed = match base.content.last() { - Some(ContentBlock::Text { text }) => { - let text = text.clone(); - base.content.pop(); - text - } - _ => String::new(), - }; - self.base = Some(base); - self.open = Some(OpenBlock::Text { - seed, - deltas: String::new(), - }); - self.saw_content = true; - } - - fn start_thinking(&mut self, partial: &AssistantMessage) { - let mut base = partial.clone(); - let seed = match base.content.last() { - Some(ContentBlock::Thinking { text, .. }) => { - let text = text.clone(); - base.content.pop(); - text - } - _ => String::new(), - }; - self.base = Some(base); - self.open = Some(OpenBlock::Thinking { - seed, - deltas: String::new(), - }); - self.saw_content = true; - } - - fn validate_boundary( - &self, - partial: &AssistantMessage, - expected_kind: BoundaryKind, - ordinal: u64, - ) -> anyhow::Result<()> { - let Some(open) = &self.open else { - return Ok(()); - }; - if open.kind() != expected_kind { - anyhow::bail!("generation {ordinal}: stream block ended with the wrong frame type"); - } - let reconstructed = self.reconstructed_content(ordinal)?; - let agrees = match expected_kind { - BoundaryKind::Thinking => { - content_equal_ignoring_thinking_signatures(&reconstructed, &partial.content) - } - BoundaryKind::Text | BoundaryKind::FunctionCall => reconstructed == partial.content, - }; - if !agrees { - anyhow::bail!( - "generation {ordinal}: streamed deltas disagree with the block-end snapshot" - ); - } - Ok(()) - } - - fn reconstructed_content(&self, ordinal: u64) -> anyhow::Result> { - let mut content = self - .base - .as_ref() - .map(|base| base.content.clone()) - .unwrap_or_default(); - match &self.open { - None => {} - Some(OpenBlock::Text { seed, deltas }) => content.push(ContentBlock::Text { - text: format!("{seed}{deltas}"), - }), - Some(OpenBlock::Thinking { seed, deltas }) => { - content.push(ContentBlock::Thinking { - text: format!("{seed}{deltas}"), - signature: None, - }); - } - Some(OpenBlock::FunctionCall { id, arguments }) => { - let parsed = serde_json::from_str::(arguments) - .map_err(|error| { - anyhow::anyhow!( - "generation {ordinal}: streamed function-call arguments are not complete JSON: {error}" - ) - })?; - if !parsed.is_object() { - anyhow::bail!( - "generation {ordinal}: streamed function-call arguments must be an object" - ); - } - let Some(ContentBlock::FunctionCall { - id: base_id, - arguments: base_arguments, - .. - }) = content - .iter_mut() - .rev() - .find(|block| matches!(block, ContentBlock::FunctionCall { .. })) - else { - anyhow::bail!( - "generation {ordinal}: function-call deltas require a start snapshot" - ); - }; - if !id.is_empty() && base_id != id { - anyhow::bail!( - "generation {ordinal}: streamed function-call id disagrees with its snapshot" - ); - } - *base_arguments = parsed; - } - } - Ok(content) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum BoundaryKind { - Text, - Thinking, - FunctionCall, -} - -impl OpenBlock { - fn kind(&self) -> BoundaryKind { - match self { - OpenBlock::Text { .. } => BoundaryKind::Text, - OpenBlock::Thinking { .. } => BoundaryKind::Thinking, - OpenBlock::FunctionCall { .. } => BoundaryKind::FunctionCall, - } - } -} - -fn content_equal_allowing_pending_thinking_signature( - reconstructed: &[ContentBlock], - terminal: &[ContentBlock], -) -> bool { - content_equal_ignoring_thinking_signatures(reconstructed, terminal) -} - -fn content_equal_ignoring_thinking_signatures( - left: &[ContentBlock], - right: &[ContentBlock], -) -> bool { - let strip = |blocks: &[ContentBlock]| { - blocks - .iter() - .cloned() - .map(|block| match block { - ContentBlock::Thinking { text, .. } => ContentBlock::Thinking { - text, - signature: None, - }, - block => block, - }) - .collect::>() - }; - strip(left) == strip(right) -} diff --git a/harness/evals/integration/src/fixtures/tests.rs b/harness/evals/integration/src/fixtures/tests.rs index 77f913b28..9303d4a54 100644 --- a/harness/evals/integration/src/fixtures/tests.rs +++ b/harness/evals/integration/src/fixtures/tests.rs @@ -1,350 +1,55 @@ -use std::collections::BTreeSet; - -use serde_json::json; - use super::*; -use crate::types::script::{JsonMatcherV1, RouterScriptV1}; - -fn minimal_script(mutate: impl FnOnce(&mut serde_json::Value)) -> serde_json::Value { - let match_ = serde_json::to_value(crate::types::script::GenerationMatchV1::uniform( - JsonMatcherV1::Absent, - )) - .unwrap(); - let mut script = json!({ - "schema_version": "1", - "scenario_id": "T", - "model": { - "id": "m", "provider": "p", - "context_window": 1000, "max_output_tokens": 100 - }, - "generations": [{ - "ordinal": 1, - "match": match_, - "frames": [{ - "type": "done", - "message": { - "role": "assistant", "content": [], "stop_reason": "end", - "model": "m", "provider": "p", "timestamp": 1 - } - }], - "response": { "ok": true, "provider": "p", "model": "m", "stop_reason": "end" } - }] - }); - mutate(&mut script); - script -} - -fn validate(value: serde_json::Value) -> anyhow::Result<()> { - let script: RouterScriptV1 = serde_json::from_value(value)?; - validate_script(&script) -} - -/// Full anyhow chain (`{:#}`): plain Display shows only the outermost -/// context, hiding the root-cause text the assertions look for. -fn error_chain(value: serde_json::Value) -> String { - format!("{:#}", validate(value).unwrap_err()) -} - -#[test] -fn a_wellformed_script_validates() { - validate(minimal_script(|_| {})).unwrap(); -} - -#[test] -fn duplicate_ordinals_are_rejected() { - let script = minimal_script(|script| { - let generation = script["generations"][0].clone(); - script["generations"] - .as_array_mut() - .unwrap() - .push(generation); - }); - assert!(error_chain(script).contains("duplicate")); -} - -#[test] -fn missing_terminal_frame_is_rejected() { - let script = minimal_script(|script| { - script["generations"][0]["frames"] = json!([{ "type": "ping" }]); - }); - assert!(error_chain(script).contains("no terminal")); -} - -#[test] -fn terminal_frame_must_be_last() { - let script = minimal_script(|script| { - let done = script["generations"][0]["frames"][0].clone(); - script["generations"][0]["frames"] = json!([done, { "type": "ping" }]); - }); - assert!(error_chain(script).contains("not the last")); -} - -#[test] -fn response_frame_disagreement_is_rejected() { - let script = minimal_script(|script| { - script["generations"][0]["response"]["ok"] = json!(false); - }); - assert!(error_chain(script).contains("ok:false")); - - let script = minimal_script(|script| { - script["generations"][0]["response"]["stop_reason"] = json!("length"); - }); - assert!(error_chain(script).contains("disagrees")); - - let script = minimal_script(|script| { - script["generations"][0]["response"]["provider"] = json!("other"); - }); - assert!(error_chain(script).contains("provider")); - - let script = minimal_script(|script| { - script["generations"][0]["response"]["usage"] = json!({ "input": 1 }); - }); - assert!(error_chain(script).contains("usage")); -} - -#[test] -fn invalid_matchers_and_normalizers_are_rejected() { - let script = minimal_script(|script| { - script["generations"][0]["match"]["model"] = json!({ "mode": "regex", "pattern": "(" }); - }); - assert!(validate(script).is_err()); - - // `replace` left the wire contract with its last emitter; the schema now - // rejects it as an unknown operation. - let script = minimal_script(|script| { - script["generations"][0]["match"]["messages"] = json!({ - "mode": "exact", "expected": [], - "normalize": [{ "pointer": "/0/x", "operation": "replace" }] - }); - }); - assert!(error_chain(script).contains("replace")); - - let script = minimal_script(|script| { - script["generations"][0]["match"]["messages"] = json!({ - "mode": "exact", "expected": [], - "normalize": [{ "pointer": "no-slash", "operation": "delete" }] - }); - }); - assert!(validate(script).is_err()); -} - -#[test] -fn unknown_fields_and_bad_sha256_are_rejected_by_schema() { - let script = minimal_script(|script| { - script["generations"][0]["surprise"] = json!(true); - }); - assert!(validate(script).is_err()); - - let script = minimal_script(|script| { - script["generations"][0]["match"]["system_prompt"] = - json!({ "mode": "sha256", "expected": "nothex" }); - }); - assert!(error_chain(script).contains("64 hex")); -} #[test] -fn removed_barriers_are_rejected_as_unknown_fields() { - let script = minimal_script(|script| { - script["generations"][0]["barriers"] = - json!([{ "before_frame": 5, "id": "b", "timeout_ms": 100 }]); - }); - assert!(error_chain(script).contains("barrier")); -} - -/// Both delta wire forms are valid fixtures: the slim delta (no -/// `partial`) the router emits today, and the legacy fat delta carrying -/// a full snapshot. -#[test] -fn slim_and_fat_deltas_both_validate() { - let slim = minimal_script(|script| { - let mut done = script["generations"][0]["frames"][0].clone(); - done["message"]["content"] = json!([{ "type": "text", "text": "x" }]); - script["generations"][0]["frames"] = json!([{ "type": "text_delta", "delta": "x" }, done]); - }); - validate(slim).unwrap(); - - let fat = minimal_script(|script| { - let mut done = script["generations"][0]["frames"][0].clone(); - done["message"]["content"] = json!([{ "type": "text", "text": "x" }]); - let partial = done["message"].clone(); - script["generations"][0]["frames"] = - json!([{ "type": "text_delta", "partial": partial, "delta": "x" }, done]); - }); - validate(fat).unwrap(); -} - -#[test] -fn streamed_text_must_agree_with_terminal_message() { - let script = minimal_script(|script| { - let mut done = script["generations"][0]["frames"][0].clone(); - done["message"]["content"] = json!([{ "type": "text", "text": "different" }]); - script["generations"][0]["frames"] = - json!([{ "type": "text_delta", "delta": "streamed" }, done]); - }); - assert!(error_chain(script).contains("streamed content disagrees")); -} - -#[test] -fn block_end_cannot_hide_incorrect_text_or_thinking_deltas() { - for (start_type, delta_type, end_type, block_type) in [ - ("text_start", "text_delta", "text_end", "text"), - ( - "thinking_start", - "thinking_delta", - "thinking_end", - "thinking", - ), - ] { - let script = minimal_script(|script| { - let mut done = script["generations"][0]["frames"][0].clone(); - done["message"]["content"] = json!([{ "type": block_type, "text": "authoritative" }]); - let mut start = done["message"].clone(); - start["content"] = json!([{ "type": block_type, "text": "" }]); - let end = done["message"].clone(); - script["generations"][0]["frames"] = json!([ - { "type": start_type, "partial": start }, - { "type": delta_type, "delta": "contradictory" }, - { "type": end_type, "partial": end }, - done - ]); - }); - assert!( - error_chain(script).contains("deltas disagree"), - "{block_type} stream was accepted" - ); - } -} - -#[test] -fn function_call_end_cannot_hide_incorrect_argument_deltas() { - let script = minimal_script(|script| { - let mut done = script["generations"][0]["frames"][0].clone(); - done["message"]["stop_reason"] = json!("function_call"); - done["message"]["content"] = json!([{ - "type": "function_call", - "id": "call-1", - "function_id": "run::target", - "arguments": { "value": "authoritative" } - }]); - script["generations"][0]["response"]["stop_reason"] = json!("function_call"); - let mut start = done["message"].clone(); - start["content"][0]["arguments"] = json!({}); - let end = done["message"].clone(); - script["generations"][0]["frames"] = json!([ - { "type": "functioncall_start", "partial": start }, - { - "type": "functioncall_delta", - "id": "call-1", - "delta": "{\"value\":\"contradictory\"}" - }, - { "type": "functioncall_end", "partial": end }, - done - ]); - }); - assert!(error_chain(script).contains("deltas disagree")); -} - -fn registered_text_scenario(slug: &str, id: &str) -> crate::scenarios::RegisteredScenario { - crate::scenarios::RegisteredScenario { - slug: slug.to_string(), - authored: crate::expand::scenario_template( - id, - "A fixture selection test.", - crate::expand::ScenarioTemplateKind::Text, - ), - driver: crate::scenarios::ScenarioDriver::Direct, - verify: |_| Ok(()), - } -} - -#[test] -fn all_selector_with_no_runnable_scenario_is_an_error() { - let empty_error = super::discovery::select_fixtures(Vec::new(), "all", true).unwrap_err(); - assert!(format!("{empty_error:#}").contains("matched no registered scenario")); - - let mut quarantined = registered_text_scenario("only-quarantined", "E2E-Q"); - quarantined.authored.quarantine = true; - let error = super::discovery::select_fixtures(vec![quarantined], "all", false).unwrap_err(); - assert!(format!("{error:#}").contains("no non-quarantined registered scenario")); +fn all_selection_returns_the_three_checked_in_fixtures() { + let fixtures = scenario_fixtures("all").unwrap(); + let ids = fixtures + .iter() + .map(|fixture| fixture.scenario.id.as_str()) + .collect::>(); + assert_eq!( + ids, + std::collections::BTreeSet::from(["E2E-001", "E2E-002", "UI-001"]) + ); + assert_eq!( + fixtures + .iter() + .filter(|fixture| fixture.driver == crate::scenarios::ScenarioDriver::Direct) + .count(), + 2 + ); } #[test] -fn explicit_selection_isolated_from_unrelated_invalid_fixtures() { - // Well-typed but semantically broken: the function call resolves no - // registered alias, so only compilation can reject it. - let mut invalid = registered_text_scenario("a-invalid", "E2E-INVALID"); - invalid.authored.router.generations[0].reply = - crate::types::scenario::RouterReplyV1::FunctionCall { - id: None, - function: "missing".to_string(), - arguments: json!({}), - usage: None, - }; - let registered = vec![ - invalid, - registered_text_scenario("z-selected", "E2E-SELECTED"), - ]; - - let by_slug = - super::discovery::select_fixtures(registered.clone(), "z-selected", false).unwrap(); - assert_eq!(by_slug[0].scenario.id, "E2E-SELECTED"); - let by_id = - super::discovery::select_fixtures(registered.clone(), "E2E-SELECTED", false).unwrap(); - assert_eq!(by_id[0].scenario.id, "E2E-SELECTED"); - assert!(super::discovery::select_fixtures(registered, "all", true).is_err()); +fn explicit_selection_accepts_slug_or_id() { + assert_eq!( + scenario_fixtures("streamed-text").unwrap()[0].scenario.id, + "E2E-001" + ); + assert_eq!( + scenario_fixtures("E2E-002").unwrap()[0].slug, + "exactly-once-function" + ); + assert!(scenario_fixtures("missing").is_err()); } #[test] -fn duplicate_identities_are_rejected_for_every_selector() { - let registered = vec![ - registered_text_scenario("first", "E2E-DUPLICATE"), - registered_text_scenario("second", "E2E-DUPLICATE"), - ]; - - for selector in ["all", "E2E-DUPLICATE", "first"] { - let error = - super::discovery::select_fixtures(registered.clone(), selector, true).unwrap_err(); - assert!( - format!("{error:#}").contains("duplicate scenario id"), - "{selector}: {error:#}" - ); - } +fn duplicate_identities_are_rejected() { + let mut fixtures = crate::scenarios::all(); + fixtures[1].scenario.id = fixtures[0].scenario.id.clone(); + let error = super::discovery::select_fixtures(fixtures, "all").unwrap_err(); + assert!(format!("{error:#}").contains("duplicate scenario id")); - let duplicate_slug = vec![ - registered_text_scenario("twice", "E2E-FIRST"), - registered_text_scenario("twice", "E2E-SECOND"), - ]; - let error = super::discovery::select_fixtures(duplicate_slug, "all", true).unwrap_err(); + let mut fixtures = crate::scenarios::all(); + fixtures[1].slug = fixtures[0].slug.clone(); + let error = super::discovery::select_fixtures(fixtures, "all").unwrap_err(); assert!(format!("{error:#}").contains("duplicate scenario slug")); } #[test] -fn all_selection_can_include_quarantines_for_validation() { - let all = scenario_fixtures("all", true).unwrap(); - let runnable = scenario_fixtures("all", false).unwrap(); - let all_ids = all - .iter() - .map(|fixture| fixture.scenario.id.as_str()) - .collect::>(); - let runnable_ids = runnable - .iter() - .map(|fixture| fixture.scenario.id.as_str()) - .collect::>(); - let quarantined_ids = all - .iter() - .filter(|fixture| fixture.quarantine) - .map(|fixture| fixture.scenario.id.as_str()) - .collect::>(); - let partition = runnable_ids - .union(&quarantined_ids) - .copied() - .collect::>(); - - assert!( - !all_ids.is_empty(), - "expected at least one checked-in scenario" - ); - assert!(runnable_ids.is_disjoint(&quarantined_ids)); - assert_eq!(partition, all_ids); - assert!(scenario_fixtures("crash-recovery-507", false).unwrap()[0].quarantine); +fn malformed_terminal_sequence_is_rejected() { + let mut fixture = crate::scenarios::all().remove(0); + fixture.script.generations[0].frames.pop(); + let error = fixture.validate().unwrap_err(); + assert!(format!("{error:#}").contains("terminal frame")); } diff --git a/harness/evals/integration/src/main.rs b/harness/evals/integration/src/main.rs index 2fb5579b8..fb460ab68 100644 --- a/harness/evals/integration/src/main.rs +++ b/harness/evals/integration/src/main.rs @@ -3,9 +3,8 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use clap::{Args, Parser, Subcommand}; -use harness_integration::expand::render_compiled; use harness_integration::fixtures::{scenario_fixtures, ScenarioFixture}; -use harness_integration::scenario::{observe_scenario, run_scenario}; +use harness_integration::scenario::{playground_scenario, run_scenario}; use harness_integration::scenarios::ScenarioDriver; use harness_integration::stack::StackBins; use harness_integration::types::scenario::Classification; @@ -22,15 +21,12 @@ struct Cli { #[derive(Debug, Subcommand)] enum Command { - /// Run one scenario or every non-quarantined scenario. + /// Run one direct scenario or every direct scenario. Run(RunArgs), - /// Compile and validate every registered scenario without booting a - /// stack. + /// Validate registered fixtures without booting a stack. Validate(SelectionArgs), - /// Print one deterministic, fully expanded compiled scenario. - Render(RenderArgs), - /// Boot one armed stack for a Playwright UI observe test. - Observe(ObserveArgs), + /// Boot one stack and Console for manual or Playwright stimulus. + Playground(PlaygroundArgs), } #[derive(Debug, Args)] @@ -66,16 +62,20 @@ struct RunArgs { } #[derive(Debug, Args)] -struct ObserveArgs { +struct PlaygroundArgs { #[command(flatten)] selection: SelectionArgs, #[command(flatten)] bins: StackBinArgs, - /// File atomically published after the stack is armed and ready. + /// Production Console binary to run against the isolated stack. #[arg(long)] - ready_file: PathBuf, + console_bin: PathBuf, + + /// Optional file atomically published after the Console is ready. + #[arg(long)] + ready_file: Option, /// Root directory for run artifacts. #[arg(long, default_value = "target/console-e2e")] @@ -89,12 +89,6 @@ struct SelectionArgs { scenario: String, } -#[derive(Debug, Args)] -struct RenderArgs { - /// Scenario id or slug. - scenario: String, -} - fn parse_worker_bin(raw: &str) -> Result<(String, PathBuf), String> { let (name, path) = raw .split_once('=') @@ -121,8 +115,7 @@ async fn dispatch(cli: Cli) -> i32 { let result = match cli.command { Command::Run(args) => return run(args).await, Command::Validate(args) => validate(args), - Command::Render(args) => render(args), - Command::Observe(args) => return observe(args).await, + Command::Playground(args) => return playground(args).await, }; match result { Ok(message) => { @@ -139,7 +132,7 @@ async fn dispatch(cli: Cli) -> i32 { } async fn run(args: RunArgs) -> i32 { - let mut fixtures = match load_fixtures(&args.selection, false) { + let mut fixtures = match load_fixtures(&args.selection) { Ok(fixtures) => fixtures, Err(error) => { eprintln!("runner_error: {error:#}"); @@ -153,7 +146,7 @@ async fn run(args: RunArgs) -> i32 { .any(|fixture| fixture.driver != ScenarioDriver::Direct) { eprintln!( - "runner_error: scenario {:?} is driven by Observe; use `observe`", + "runner_error: scenario {:?} is driven by Playground; use `playground`", args.selection.scenario ); return 3; @@ -195,16 +188,16 @@ async fn run(args: RunArgs) -> i32 { exit_code } -async fn observe(args: ObserveArgs) -> i32 { +async fn playground(args: PlaygroundArgs) -> i32 { if args.selection.scenario == "all" { - eprintln!("runner_error: observe requires one scenario id or slug"); + eprintln!("runner_error: playground requires one scenario id or slug"); return 3; } - let fixture = match load_fixtures(&args.selection, true).and_then(|fixtures| { + let fixture = match load_fixtures(&args.selection).and_then(|fixtures| { fixtures .into_iter() .next() - .context("observe selector returned no scenario") + .context("playground selector returned no scenario") }) { Ok(fixture) => fixture, Err(error) => { @@ -212,13 +205,21 @@ async fn observe(args: ObserveArgs) -> i32 { return 3; } }; - let bins = match resolve_stack_bins(&args.bins) { + let mut bins = match resolve_stack_bins(&args.bins) { Ok(bins) => bins, Err(error) => { eprintln!("runner_error: {error:#}"); return 3; } }; + let console_bin = match absolute_binary("console", &args.console_bin) { + Ok(bin) => bin, + Err(error) => { + eprintln!("runner_error: {error:#}"); + return 3; + } + }; + bins.console = Some(console_bin.clone()); let artifacts_dir = match prepare_artifacts_dir(&args.artifacts_dir) { Ok(dir) => dir, Err(error) => { @@ -227,7 +228,14 @@ async fn observe(args: ObserveArgs) -> i32 { } }; - let outcome = observe_scenario(&bins, &fixture, &artifacts_dir, &args.ready_file).await; + let outcome = playground_scenario( + &bins, + &console_bin, + &fixture, + &artifacts_dir, + args.ready_file.as_deref(), + ) + .await; println!( "{}: {} — run {} ({} ms), artifacts: {}", fixture.scenario.id, @@ -240,31 +248,12 @@ async fn observe(args: ObserveArgs) -> i32 { } fn validate(args: SelectionArgs) -> anyhow::Result { - let fixtures = load_fixtures(&args, true)?; + let fixtures = load_fixtures(&args)?; Ok(format!("{} scenario fixture(s) valid", fixtures.len())) } -fn render(args: RenderArgs) -> anyhow::Result { - anyhow::ensure!( - args.scenario != "all", - "render requires one scenario id or slug" - ); - let selection = SelectionArgs { - scenario: args.scenario, - }; - let fixtures = load_fixtures(&selection, true)?; - let fixture = fixtures - .into_iter() - .next() - .context("render selector returned no scenario")?; - render_compiled(&fixture.compiled()) -} - -fn load_fixtures( - selection: &SelectionArgs, - include_quarantined: bool, -) -> anyhow::Result> { - scenario_fixtures(&selection.scenario, include_quarantined) +fn load_fixtures(selection: &SelectionArgs) -> anyhow::Result> { + scenario_fixtures(&selection.scenario) } fn classification_str(classification: Classification) -> &'static str { @@ -296,6 +285,7 @@ fn resolve_stack_bins(args: &StackBinArgs) -> anyhow::Result { let bins = StackBins { engine: absolute_binary("engine", engine)?, harness: absolute_binary("harness", harness)?, + console: None, workers: args .worker_bins .iter() diff --git a/harness/evals/integration/src/matcher.rs b/harness/evals/integration/src/matcher.rs index d688a822e..d53248d4e 100644 --- a/harness/evals/integration/src/matcher.rs +++ b/harness/evals/integration/src/matcher.rs @@ -39,10 +39,6 @@ pub fn evaluate(field: &str, matcher: &JsonMatcherV1, actual: Option<&Value>) -> None => pass, Some(v) => fail(format!("expected absent, got {}", head(v))), }, - JsonMatcherV1::Present => match actual { - Some(_) => pass, - None => fail("expected present, field is absent".into()), - }, JsonMatcherV1::Regex { pattern } => { let Some(actual) = actual else { return fail("regex matcher: field is absent".into()); @@ -190,42 +186,20 @@ fn split_pointer(pointer: &str) -> (&str, String) { (&pointer[..idx], token) } -/// Array semantics for structural subset comparisons. Objects are always -/// recursive subsets; callers must choose whether arrays may have an -/// unmatched suffix or must match in full. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ArrayPolicy { - Exact, - Prefix, -} - -pub fn subset_with_array_policy( - expected: &Value, - actual: &Value, - arrays: ArrayPolicy, -) -> Option { - subset_mismatch_with_arrays(expected, actual, "", arrays) -} - /// `None` when `expected` is a positional subset of `actual`; otherwise a /// description of the first mismatch. fn subset_mismatch(expected: &Value, actual: &Value, path: &str) -> Option { - subset_mismatch_with_arrays(expected, actual, path, ArrayPolicy::Prefix) + subset_mismatch_with_arrays(expected, actual, path) } -fn subset_mismatch_with_arrays( - expected: &Value, - actual: &Value, - path: &str, - arrays: ArrayPolicy, -) -> Option { +fn subset_mismatch_with_arrays(expected: &Value, actual: &Value, path: &str) -> Option { match (expected, actual) { (Value::Object(exp), Value::Object(act)) => { for (k, ev) in exp { match act.get(k) { Some(av) => { if let Some(detail) = - subset_mismatch_with_arrays(ev, av, &format!("{path}/{k}"), arrays) + subset_mismatch_with_arrays(ev, av, &format!("{path}/{k}")) { return Some(detail); } @@ -236,11 +210,7 @@ fn subset_mismatch_with_arrays( None } (Value::Array(exp), Value::Array(act)) => { - let wrong_length = match arrays { - ArrayPolicy::Exact => exp.len() != act.len(), - ArrayPolicy::Prefix => exp.len() > act.len(), - }; - if wrong_length { + if exp.len() > act.len() { return Some(format!( "subset: expected {} array elements at {path}, got {}", exp.len(), @@ -249,7 +219,7 @@ fn subset_mismatch_with_arrays( } for (i, ev) in exp.iter().enumerate() { if let Some(detail) = - subset_mismatch_with_arrays(ev, &act[i], &format!("{path}/{i}"), arrays) + subset_mismatch_with_arrays(ev, &act[i], &format!("{path}/{i}")) { return Some(detail); } @@ -295,11 +265,9 @@ mod tests { } #[test] - fn absent_and_present() { + fn absent_distinguishes_null_from_missing() { assert!(evaluate("f", &JsonMatcherV1::Absent, None).passed); assert!(!evaluate("f", &JsonMatcherV1::Absent, Some(&json!(null))).passed); - assert!(evaluate("f", &JsonMatcherV1::Present, Some(&json!(null))).passed); - assert!(!evaluate("f", &JsonMatcherV1::Present, None).passed); } /// The truncation cut point lands mid-emoji: 199 leading bytes (quote + diff --git a/harness/evals/integration/src/process/child.rs b/harness/evals/integration/src/process/child.rs index 2a877e6b1..49e018a6e 100644 --- a/harness/evals/integration/src/process/child.rs +++ b/harness/evals/integration/src/process/child.rs @@ -9,7 +9,6 @@ use nix::sys::wait::waitpid; use nix::unistd::Pid; pub(super) const REAP_INTERVAL: Duration = Duration::from_millis(25); -const IMMEDIATE_KILL_BUDGET: Duration = Duration::from_secs(2); /// Tracks who owns the final wait for the direct child. /// @@ -60,32 +59,6 @@ impl SupervisedChild { self.child.try_wait() } - pub async fn kill_now(&mut self) -> anyhow::Result<()> { - self.signal_tree(Signal::SIGKILL)?; - let deadline = tokio::time::Instant::now() + IMMEDIATE_KILL_BUDGET; - loop { - match self.poll_reap() { - Ok(true) => return Ok(()), - Ok(false) if tokio::time::Instant::now() < deadline => { - tokio::time::sleep(REAP_INTERVAL).await; - } - Ok(false) => { - let _ = self.child.kill(); - self.defer_reap(); - anyhow::bail!( - "{} did not reap within {}ms after SIGKILL", - self.name, - IMMEDIATE_KILL_BUDGET.as_millis() - ); - } - Err(error) => { - self.defer_reap(); - return Err(error).with_context(|| format!("reaping {}", self.name)); - } - } - } - } - pub(super) fn signal_tree(&self, signal: Signal) -> anyhow::Result<()> { let group = self.signal_group(signal); let direct = match kill(Pid::from_raw(self.child.id() as i32), signal) { diff --git a/harness/evals/integration/src/process/supervisor.rs b/harness/evals/integration/src/process/supervisor.rs index 37cb0af7d..8490ac59f 100644 --- a/harness/evals/integration/src/process/supervisor.rs +++ b/harness/evals/integration/src/process/supervisor.rs @@ -62,6 +62,7 @@ impl ProcessSupervisor { Ok(pid) } + #[cfg(test)] pub fn remove(&mut self, name: &str) -> Option { let index = self .children diff --git a/harness/evals/integration/src/process/tests.rs b/harness/evals/integration/src/process/tests.rs index 71a1550a4..d63f1b44d 100644 --- a/harness/evals/integration/src/process/tests.rs +++ b/harness/evals/integration/src/process/tests.rs @@ -42,26 +42,6 @@ fn drop_cleans_up_a_partially_started_supervisor() { ); } -#[tokio::test] -async fn kill_now_is_idempotent_after_a_synchronous_reap() { - let dir = tempfile::tempdir().unwrap(); - let mut supervisor = ProcessSupervisor::new(Duration::from_millis(100)); - let running = ProcessSpec::new( - "running", - "/bin/sh", - dir.path(), - dir.path().join("running.out"), - dir.path().join("running.err"), - ) - .args(["-c", "exec sleep 60"]); - let pid = supervisor.spawn(running).unwrap(); - let mut child = supervisor.remove("running").unwrap(); - - child.kill_now().await.unwrap(); - assert!(!process_is_running(pid)); - child.kill_now().await.unwrap(); -} - #[tokio::test] async fn observing_a_leader_exit_does_not_abandon_its_descendants() { let dir = tempfile::tempdir().unwrap(); diff --git a/harness/evals/integration/src/recorder/service.rs b/harness/evals/integration/src/recorder/service.rs index c0ff84073..f39083ab3 100644 --- a/harness/evals/integration/src/recorder/service.rs +++ b/harness/evals/integration/src/recorder/service.rs @@ -1,9 +1,7 @@ //! SDK-facing recorder service and controlled-function registration. -use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use iii_sdk::errors::Error; use iii_sdk::protocol::RegisterTriggerInput; @@ -23,44 +21,6 @@ pub struct Recorder { client: Client, store: Arc, event_notify: Arc, - response_gates: Arc>>>, -} - -pub(super) struct ResponseGate { - hold_at: u64, - calls: AtomicU64, - released: AtomicBool, - notify: tokio::sync::Notify, -} - -impl ResponseGate { - pub(super) fn new(hold_at: u64) -> Self { - Self { - hold_at, - calls: AtomicU64::new(0), - released: AtomicBool::new(false), - notify: tokio::sync::Notify::new(), - } - } - - pub(super) async fn wait_if_selected(&self) { - let ordinal = self.calls.fetch_add(1, Ordering::AcqRel) + 1; - if ordinal != self.hold_at { - return; - } - loop { - let notified = self.notify.notified(); - if self.released.load(Ordering::Acquire) { - return; - } - notified.await; - } - } - - pub(super) fn release(&self) { - self.released.store(true, Ordering::Release); - self.notify.notify_waiters(); - } } impl Recorder { @@ -71,7 +31,6 @@ impl Recorder { client, store, event_notify: Arc::new(tokio::sync::Notify::new()), - response_gates: Arc::new(Mutex::new(BTreeMap::new())), }; recorder.register_lifecycle(); Ok(recorder) @@ -109,54 +68,19 @@ impl Recorder { /// Configure and register the run-scoped controlled functions directly. pub fn configure(&self, run_id: &str, config: &RecorderConfigV1) -> anyhow::Result<()> { let prefix = format!("{run_id}::"); - let mut function_ids = BTreeSet::new(); - for declared in controlled_functions(config) { - anyhow::ensure!( - declared.function_id.starts_with(&prefix), - "integration/target_scope: {} must be prefixed by {prefix}", - declared.function_id - ); - anyhow::ensure!( - function_ids.insert(declared.function_id.as_str()), - "integration/target_duplicate: {} is declared more than once", - declared.function_id - ); - } + anyhow::ensure!( + config.target.function_id.starts_with(&prefix), + "integration/target_scope: {} must be prefixed by {prefix}", + config.target.function_id + ); self.store.configure(run_id)?; - let mut gates = self - .response_gates - .lock() - .map_err(|_| anyhow::anyhow!("recorder response gate lock poisoned"))?; - gates.clear(); - for declared in controlled_functions(config) { - let gate = declared - .hold_response_at - .map(|hold_at| Arc::new(ResponseGate::new(hold_at))); - if let Some(gate) = &gate { - gates.insert(declared.function_id.clone(), Arc::clone(gate)); - } - register_controlled_function( - self.client.inner(), - &self.store, - declared, - gate, - Arc::clone(&self.event_notify), - ); - } - Ok(()) - } - - /// Release a private response gate after the runner has applied a fault. - pub fn release_response(&self, function_id: &str) -> anyhow::Result<()> { - let gates = self - .response_gates - .lock() - .map_err(|_| anyhow::anyhow!("recorder response gate lock poisoned"))?; - let gate = gates - .get(function_id) - .ok_or_else(|| anyhow::anyhow!("no response gate configured for {function_id}"))?; - gate.release(); + register_controlled_function( + self.client.inner(), + &self.store, + &config.target, + Arc::clone(&self.event_notify), + ); Ok(()) } @@ -198,32 +122,16 @@ impl Recorder { /// (there are none in v1, but the filter is part of the contract) never /// deliver here. pub async fn bind_lifecycle(&self, trigger_type: &str, session_id: &str) -> anyhow::Result<()> { - self.bind( - trigger_type, - LIFECYCLE_FUNCTION_ID, - json!({ "session_id": session_id }), - ) - .await - } - - /// Create an arbitrary trigger binding on the recorder's connection - /// (scenario `bindings`, e.g. `harness::hook::pre-trigger` chains). - pub async fn bind( - &self, - trigger_type: &str, - function_id: &str, - config: Value, - ) -> anyhow::Result<()> { self.client .inner() .register_trigger(RegisterTriggerInput { trigger_type: trigger_type.to_string(), - function_id: function_id.to_string(), - config, + function_id: LIFECYCLE_FUNCTION_ID.to_string(), + config: json!({ "session_id": session_id }), metadata: None, }) .map_err(|error| { - anyhow::anyhow!("binding {trigger_type} -> {function_id} failed: {error}") + anyhow::anyhow!("binding {trigger_type} -> {LIFECYCLE_FUNCTION_ID} failed: {error}") })?; Ok(()) } @@ -240,18 +148,11 @@ pub(super) fn parse_lifecycle_payload(payload: Value) -> Result { serde_json::to_value(event).map_err(|error| format!("integration/lifecycle_serialize: {error}")) } -fn controlled_functions(config: &RecorderConfigV1) -> impl Iterator { - std::iter::once(&config.target).chain(config.extra_functions.iter()) -} - -/// Register one declared controlled function: verbatim description/schema, -/// durable append per call, and the declared response (after an optional -/// compiler-owned fault gate). +/// Register the declared controlled function with durable recording. fn register_controlled_function( iii: &iii_sdk::IIIClient, store: &Arc, declared: &RecorderTargetV1, - response_gate: Option>, event_notify: Arc, ) { let response = declared.response.clone(); @@ -265,7 +166,6 @@ fn register_controlled_function( let event_notify = Arc::clone(&event_notify); let response = response.clone(); let function_id = handler_function_id.clone(); - let response_gate = response_gate.clone(); async move { append_handler_event( &store, @@ -275,9 +175,6 @@ fn register_controlled_function( "integration/target_append", )?; event_notify.notify_waiters(); - if let Some(gate) = response_gate { - gate.wait_if_selected().await; - } Ok::(response) } }) diff --git a/harness/evals/integration/src/recorder/tests.rs b/harness/evals/integration/src/recorder/tests.rs index d6a5e7a90..7f717c186 100644 --- a/harness/evals/integration/src/recorder/tests.rs +++ b/harness/evals/integration/src/recorder/tests.rs @@ -2,7 +2,7 @@ use std::path::Path; use serde_json::json; -use super::service::{parse_lifecycle_payload, strip_engine_fields, ResponseGate}; +use super::service::{parse_lifecycle_payload, strip_engine_fields}; use super::store::EventStore; use crate::types::recorder::RecorderEventKind; @@ -151,27 +151,6 @@ fn lifecycle_contract_strips_only_engine_fields_and_rejects_unknown_shape() { assert!(parse_lifecycle_payload(unknown_nested).is_err()); } -#[tokio::test] -async fn response_gate_holds_only_the_selected_call_ordinal() { - let gate = std::sync::Arc::new(ResponseGate::new(2)); - gate.wait_if_selected().await; - - let waiting = { - let gate = std::sync::Arc::clone(&gate); - tokio::spawn(async move { gate.wait_if_selected().await }) - }; - tokio::task::yield_now().await; - assert!(!waiting.is_finished(), "the selected call must remain held"); - - gate.release(); - tokio::time::timeout(std::time::Duration::from_secs(1), waiting) - .await - .expect("released gate should wake") - .expect("gate task should not panic"); - - gate.wait_if_selected().await; -} - #[cfg(target_os = "linux")] #[test] fn write_errors_are_returned_without_acknowledging_the_event() { diff --git a/harness/evals/integration/src/runtime.rs b/harness/evals/integration/src/runtime.rs index 1937c836e..e04b5b7b9 100644 --- a/harness/evals/integration/src/runtime.rs +++ b/harness/evals/integration/src/runtime.rs @@ -8,11 +8,8 @@ use crate::types::scenario::Classification; pub enum RunPhase { Allocate, Boot, - Probe, Arm, Send, - Fault, - Release, Await, Collect, Grade, @@ -25,11 +22,8 @@ impl std::fmt::Display for RunPhase { formatter.write_str(match self { RunPhase::Allocate => "allocate", RunPhase::Boot => "boot", - RunPhase::Probe => "probe", RunPhase::Arm => "arm", RunPhase::Send => "send", - RunPhase::Fault => "fault", - RunPhase::Release => "release", RunPhase::Await => "await", RunPhase::Collect => "collect", RunPhase::Grade => "grade", diff --git a/harness/evals/integration/src/scenario.rs b/harness/evals/integration/src/scenario.rs index 53b6c6aa9..2d73c09d6 100644 --- a/harness/evals/integration/src/scenario.rs +++ b/harness/evals/integration/src/scenario.rs @@ -1,21 +1,20 @@ -//! Scenario execution lifecycle: -//! Allocate → Boot → Arm → Send → Fault/Release → Await → -//! Collect → Grade → Teardown → Report. -//! (Observe inserts Probe/wait-start between Arm and Send, then waits for -//! observer shutdown after Await before Collect.) +//! Scenario execution lifecycle: allocate → boot → arm → send/observe → +//! await → collect → grade → teardown → report. //! //! Every phase returns [`crate::runtime::RunError`]. Classification is derived //! once after process state has been inspected. pub mod floor; -mod observe; mod phases; +mod playground; mod report; mod runner; mod state; -pub use observe::{observe_scenario, ObserveOutcome, ObserveReadyV1, ObserveResultV1}; +pub use playground::{ + playground_scenario, PlaygroundOutcome, PlaygroundReadyV1, PlaygroundResultV1, +}; pub use runner::{run_scenario, RunOutcome}; #[cfg(test)] diff --git a/harness/evals/integration/src/scenario/floor.rs b/harness/evals/integration/src/scenario/floor.rs index 39d610abf..225853352 100644 --- a/harness/evals/integration/src/scenario/floor.rs +++ b/harness/evals/integration/src/scenario/floor.rs @@ -160,7 +160,7 @@ fn generations_failure(run: &RunEvidence) -> Option { } /// Send accepted with clean flags. Skipped only when `send_response` is -/// absent (should not happen for Direct or Observe after a successful Send). +/// absent for Playground, where the Console or Playwright owns Send. fn send_flags_failure(run: &RunEvidence) -> Option { let response = run.send_response.as_ref()?; // Absent optional flags normalize to false. diff --git a/harness/evals/integration/src/scenario/phases/arm.rs b/harness/evals/integration/src/scenario/phases/arm.rs index 4ee96b812..326199b92 100644 --- a/harness/evals/integration/src/scenario/phases/arm.rs +++ b/harness/evals/integration/src/scenario/phases/arm.rs @@ -62,26 +62,6 @@ impl ScenarioRunner<'_> { ) .await .map_err(|error| RunError::setup(phase, "bind lifecycle recorder", error))?; - for binding in &scenario.bindings { - recorder - .bind( - &binding.trigger_type, - &binding.function_id, - binding.config.clone(), - ) - .await - .map_err(|error| { - RunError::setup( - phase, - format!( - "bind trigger {} to {}", - binding.trigger_type, binding.function_id - ), - error, - ) - })?; - } - self.sink_mut(phase)? .write_scenario_text( &scenario.id, diff --git a/harness/evals/integration/src/scenario/phases/completion.rs b/harness/evals/integration/src/scenario/phases/completion.rs index c1ccdd75a..1da92f790 100644 --- a/harness/evals/integration/src/scenario/phases/completion.rs +++ b/harness/evals/integration/src/scenario/phases/completion.rs @@ -25,7 +25,15 @@ impl ScenarioRunner<'_> { // recorder. Once it arrives, make one status call as the durable-state // confirmation checked by the floor. match services.recorder().wait_for_lifecycle(deadline).await { - Ok(_) => {} + Ok(event) => { + if active.turn_id.is_none() { + active.turn_id = event + .payload + .get("turn_id") + .and_then(serde_json::Value::as_str) + .map(String::from); + } + } Err(error) if deadline.is_expired() => { active.timed_out = true; tracing::error!( diff --git a/harness/evals/integration/src/scenario/phases/execution.rs b/harness/evals/integration/src/scenario/phases/execution.rs index 6d9957dae..3a082a62f 100644 --- a/harness/evals/integration/src/scenario/phases/execution.rs +++ b/harness/evals/integration/src/scenario/phases/execution.rs @@ -2,22 +2,15 @@ use std::time::Duration; use serde_json::{json, Value}; -use crate::client::{Client, DEFAULT_CALL_TIMEOUT_MS}; use crate::deadline::Deadline; -use crate::discovery; use crate::runtime::{RunError, RunErrorKind, RunPhase}; use crate::services::RunServices; -use crate::stack::Stack; -use crate::types::recorder::RecorderEventKind; -use crate::types::scenario::{CompiledScenarioV1, FaultKind}; use super::super::report::rpc_failure; use super::super::runner::ScenarioRunner; use super::super::state::{ActiveTurn, PreparedRun}; const SEND_TIMEOUT_MS: u64 = 30_000; -const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(250); -const TARGET_POLL_INTERVAL: Duration = Duration::from_millis(50); impl ScenarioRunner<'_> { pub(in crate::scenario) async fn send( @@ -57,351 +50,19 @@ impl ScenarioRunner<'_> { Ok(ActiveTurn::new(deadline, turn_id, value)) } Err(error) => { - let value = json!({ "error": error }); - self.write_artifact(&prepared.scenario.id, "send-response.json", &value, phase)?; - Err(rpc_failure( - phase, - RunErrorKind::Contract, - "harness::send failed", - error, - )) - } - } - } - - pub(in crate::scenario) async fn fault( - &mut self, - stack: &mut Stack, - services: &RunServices, - prepared: &PreparedRun, - active: &ActiveTurn, - ) -> Result<(), RunError> { - let Some(fault) = &prepared.scenario.fault else { - return Ok(()); - }; - let phase = RunPhase::Fault; - let deadline = active.deadline; - - let FaultKind::EngineSigkill = fault.kind; - let observed = deadline - .poll_until("fault trigger", TARGET_POLL_INTERVAL, || async { - let events = services.recorder().snapshot()?; - let count = events - .iter() - .filter(|event| { - event.kind == RecorderEventKind::TargetCall - && event.function_id == fault.function_id - }) - .count() as u64; - Ok((count >= fault.after_target_calls).then_some(())) - }) - .await; - if let Err(error) = observed { - services - .recorder() - .release_response(&fault.function_id) - .map_err(|release_error| { - RunError::runner( - phase, - "release controlled response gate after fault trigger failure", - release_error, - ) - })?; - let kind = if deadline.is_expired() { - RunErrorKind::Contract - } else { - RunErrorKind::Runner - }; - return Err(RunError::with_source( - phase, - kind, - format!( - "fewer than {} target calls observed before fault", - fault.after_target_calls - ), - error, - )); - } - - let kill_result = stack.kill_engine().await; - services - .recorder() - .release_response(&fault.function_id) - .map_err(|error| { - RunError::runner(phase, "release controlled response gate after fault", error) - })?; - kill_result - .map_err(|error| RunError::runner(phase, "kill engine for fault injection", error))?; - deadline - .timeout( - "fault restart delay", - tokio::time::sleep(Duration::from_millis(fault.restart_delay_ms)), - ) - .await - .map_err(|error| { - RunError::runner( - phase, - "fault restart delay exceeded scenario deadline", - error, - ) - })?; - stack.respawn_engine().map_err(|error| { - RunError::runner(phase, "respawn engine after fault injection", error) - })?; - self.restore_after_engine_restart(services, prepared, deadline) - .await?; - Ok(()) - } - - async fn restore_after_engine_restart( - &mut self, - services: &RunServices, - prepared: &PreparedRun, - deadline: Deadline, - ) -> Result<(), RunError> { - let phase = RunPhase::Fault; - let scenario = &prepared.scenario; - - discovery::wait_for_functions(services.client(), discovery::TURN_SURFACE, deadline) - .await - .map_err(|error| { - RunError::runner( - phase, - "wait for turn function surface after engine restart", - error, - ) - })?; - discovery::wait_for_trigger_types( - services.client(), - &["harness::turn-started", "harness::turn-completed"], - deadline, - ) - .await - .map_err(|error| { - RunError::runner( - phase, - "wait for harness trigger types after engine restart", - error, - ) - })?; - - let expected_bindings = expected_trigger_bindings(scenario, &self.session_id); - let registered = registered_trigger_snapshot(services.client(), deadline) - .await - .map_err(|error| { - RunError::runner( - phase, - "inspect trigger bindings after engine restart", - error, - ) - })?; - - for (index, expected) in expected_bindings.iter().enumerate() { - match registered_trigger_count(®istered, expected) { - 1 => {} - 0 if index == 0 => services - .recorder() - .bind_lifecycle( - scenario.recorder.lifecycle.trigger_type.as_str(), - &self.session_id, - ) - .await - .map_err(|error| { - RunError::runner( - phase, - "restore lifecycle binding after engine restart", - error, - ) - })?, - 0 => { - let binding = &scenario.bindings[index - 1]; - services - .recorder() - .bind( - &binding.trigger_type, - &binding.function_id, - binding.config.clone(), - ) - .await - .map_err(|error| { - RunError::runner( - phase, - format!( - "restore trigger {} -> {} after engine restart", - binding.trigger_type, binding.function_id - ), - error, - ) - })?; - } - count => { - return Err(RunError::new( - phase, - RunErrorKind::Runner, - format!( - "trigger binding {} -> {} appeared {count} times after engine restart", - expected.trigger_type, expected.function_id - ), - )); - } - } - } - Ok(()) - } - - pub(in crate::scenario) async fn release( - &mut self, - services: &RunServices, - prepared: &PreparedRun, - active: &ActiveTurn, - ) -> Result<(), RunError> { - let Some(release) = &prepared.scenario.release else { - return Ok(()); - }; - let phase = RunPhase::Release; - let deadline = active.deadline; - - deadline - .poll_until( - format!("pending call {}", release.function_call_id), - STATUS_POLL_INTERVAL, - || async { - let status = services - .client() - .call_with_deadline( - "harness::status", - json!({ "session_id": self.session_id }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await; - Ok(status.ok().and_then(|status| { - status - .get("pending_function_calls") - .and_then(Value::as_array) - .is_some_and(|calls| { - calls - .iter() - .any(|call| call.as_str() == Some(&release.function_call_id)) - }) - .then_some(()) - })) - }, - ) - .await - .map_err(|error| { - RunError::with_source( - phase, - RunErrorKind::Contract, - format!( - "call {} never appeared as pending", - release.function_call_id - ), - error, - ) - })?; - - let response = services - .client() - .call_with_deadline( - "harness::function::resolve", - json!({ - "session_id": self.session_id, - "turn_id": active.turn_id, - "function_call_id": release.function_call_id, - "action": release.action, - }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await; - match response { - Ok(value) => { self.write_artifact( &prepared.scenario.id, - "resolve-response.json", - &value, - phase, - )?; - Ok(()) - } - Err(error) => { - let value = json!({ "error": error }); - self.write_artifact( - &prepared.scenario.id, - "resolve-response.json", - &value, + "send-response.json", + &json!({ "error": error }), phase, )?; Err(rpc_failure( phase, RunErrorKind::Contract, - "harness::function::resolve failed", + "harness::send failed", error, )) } } } } - -#[derive(Debug, Clone, PartialEq)] -struct ExpectedTriggerBinding { - trigger_type: String, - function_id: String, - config: Value, -} - -fn expected_trigger_bindings( - scenario: &CompiledScenarioV1, - session_id: &str, -) -> Vec { - std::iter::once(ExpectedTriggerBinding { - trigger_type: scenario - .recorder - .lifecycle - .trigger_type - .as_str() - .to_string(), - function_id: "integration-recorder::lifecycle".to_string(), - config: json!({ "session_id": session_id }), - }) - .chain( - scenario - .bindings - .iter() - .map(|binding| ExpectedTriggerBinding { - trigger_type: binding.trigger_type.clone(), - function_id: binding.function_id.clone(), - config: binding.config.clone(), - }), - ) - .collect() -} - -async fn registered_trigger_snapshot(client: &Client, deadline: Deadline) -> anyhow::Result { - client - .call_with_deadline( - "engine::registered-triggers::list", - json!({ "include_internal": true }), - deadline, - DEFAULT_CALL_TIMEOUT_MS, - ) - .await - .map_err(anyhow::Error::msg) -} - -fn registered_trigger_count(listed: &Value, expected: &ExpectedTriggerBinding) -> usize { - listed - .get("registered_triggers") - .and_then(Value::as_array) - .or_else(|| listed.as_array()) - .into_iter() - .flatten() - .filter(|row| { - row.get("function_id").and_then(Value::as_str) == Some(expected.function_id.as_str()) - && row.get("trigger_type").and_then(Value::as_str) - == Some(expected.trigger_type.as_str()) - && row.get("config") == Some(&expected.config) - }) - .count() -} diff --git a/harness/evals/integration/src/scenario/observe.rs b/harness/evals/integration/src/scenario/playground.rs similarity index 55% rename from harness/evals/integration/src/scenario/observe.rs rename to harness/evals/integration/src/scenario/playground.rs index 796dde3df..2b73e1dbe 100644 --- a/harness/evals/integration/src/scenario/observe.rs +++ b/harness/evals/integration/src/scenario/playground.rs @@ -1,9 +1,8 @@ -//! Observe driver: armed stack for Playwright UI tests. -//! -//! The integration owns stimulus (`harness::send` after a start signal). -//! Playwright owns the Console process and DOM assertions. +//! Playground driver: the integration owns the stack and production Console; +//! a person or Playwright owns the turn stimulus. use std::collections::BTreeMap; +use std::io::Write; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -21,14 +20,14 @@ use crate::types::scenario::{Classification, CompiledSendV1}; use crate::types::script::SchemaVersion1; use super::runner::{BootedRun, ExpandedRun, ScenarioRunner}; -use super::state::PreparedRun; +use super::state::{ActiveTurn, PreparedRun}; -const START_POLL_INTERVAL: Duration = Duration::from_millis(100); const PROCESS_POLL_INTERVAL: Duration = Duration::from_millis(100); +const SHUTDOWN_COMPLETION_GRACE: Duration = Duration::from_secs(1); #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ObserveReadyV1 { +pub struct PlaygroundReadyV1 { pub schema_version: SchemaVersion1, pub run_id: String, pub scenario_id: String, @@ -37,8 +36,9 @@ pub struct ObserveReadyV1 { pub run_root: PathBuf, pub result_path: PathBuf, pub engine_url: String, - pub session: ObserveSessionV1, - pub model: ObserveModelV1, + pub console_url: String, + pub session: PlaygroundSessionV1, + pub model: PlaygroundModelV1, pub message: String, pub functions: BTreeMap, pub send: CompiledSendV1, @@ -46,66 +46,53 @@ pub struct ObserveReadyV1 { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ObserveSessionV1 { +pub struct PlaygroundSessionV1 { pub id: String, pub title: String, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ObserveModelV1 { +pub struct PlaygroundModelV1 { pub id: String, pub provider: String, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ObserveStartV1 { - pub schema_version: SchemaVersion1, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ObserveResultV1 { +pub struct PlaygroundResultV1 { pub schema_version: SchemaVersion1, pub scenario_id: String, pub classification: Classification, - /// First floor or verify failure, scrubbed of run-scoped ids. pub failure: Option, - /// Raw serialized [`crate::evidence_data::RunEvidence`] (JSON null when - /// the run failed before collection). Ids are real, so Playwright can - /// check evidence against the ready manifest. pub evidence: serde_json::Value, pub artifacts: Vec, } -pub struct ObserveOutcome { - pub result: ObserveResultV1, +pub struct PlaygroundOutcome { + pub result: PlaygroundResultV1, pub run_id: String, pub run_root: PathBuf, pub duration_ms: u64, } -pub async fn observe_scenario( +pub async fn playground_scenario( bins: &StackBins, + console_bin: &Path, fixture: &ScenarioFixture, artifacts_dir: &Path, - ready_file: &Path, -) -> ObserveOutcome { + ready_file: Option<&Path>, +) -> PlaygroundOutcome { let started = std::time::Instant::now(); - let run_id = format!("iu{}", &uuid::Uuid::new_v4().simple().to_string()[..12]); + let run_id = format!("ip{}", &uuid::Uuid::new_v4().simple().to_string()[..12]); let run_root = artifacts_dir.join(&run_id); let session_id = format!("s_{}", uuid::Uuid::new_v4().simple()); - let mut runner = ScenarioRunner::new(bins, fixture, run_id.clone(), session_id); - // The observe consumer contract is observe-ready.json / ready-file -> - // observe-result.json; the direct-run result.json/execution.json pair is - // not written here. - let mut classification = execute_observe(&mut runner, artifacts_dir, ready_file).await; + let mut classification = + execute_playground(&mut runner, console_bin, artifacts_dir, ready_file).await; let duration_ms = started.elapsed().as_millis() as u64; - - let mut result = ObserveResultV1 { + let mut result = PlaygroundResultV1 { schema_version: SchemaVersion1::V1, scenario_id: fixture.scenario.id.clone(), classification, @@ -117,11 +104,12 @@ pub async fn observe_scenario( .map(|sink| sink.paths().to_vec()) .unwrap_or_default(), }; - if let Err(error) = write_json(&run_root, &run_root.join("observe-result.json"), &result) { - tracing::error!(target: "harness_integration::scenario", "observe result failed: {error:#}"); + let result_path = run_root.join("playground-result.json"); + if let Err(error) = write_json(&run_root, &result_path, &result) { + tracing::error!(target: "harness_integration::scenario", "playground result failed: {error:#}"); classification = classification.combine(Classification::RunnerError); result.classification = classification; - let _ = write_json(&run_root, &run_root.join("observe-result.json"), &result); + let _ = write_json(&run_root, &result_path, &result); } if classification == Classification::Pass { @@ -130,7 +118,7 @@ pub async fn observe_scenario( } } - ObserveOutcome { + PlaygroundOutcome { result, run_id, run_root, @@ -138,16 +126,16 @@ pub async fn observe_scenario( } } -async fn execute_observe( +async fn execute_playground( runner: &mut ScenarioRunner<'_>, + console_bin: &Path, artifacts_dir: &Path, - ready_file: &Path, + ready_file: Option<&Path>, ) -> Classification { let ExpandedRun { paths, expanded } = match runner.expand_for_run(artifacts_dir) { Ok(expanded) => expanded, Err(classification) => return classification, }; - let mut booted = match runner.boot_prepared(paths, expanded).await { Ok(booted) => booted, Err(classification) => return classification, @@ -155,7 +143,7 @@ async fn execute_observe( let outcome = async { runner.arm_booted(&mut booted).await?; - run_observe_phases(runner, &mut booted, ready_file).await + run_playground_phases(runner, &mut booted, console_bin, ready_file).await } .await; runner @@ -168,16 +156,17 @@ async fn execute_observe( .await } -async fn run_observe_phases( +async fn run_playground_phases( runner: &mut ScenarioRunner<'_>, booted: &mut BootedRun, - ready_file: &Path, + console_bin: &Path, + ready_file: Option<&Path>, ) -> Result<(), RunError> { let stack = &mut booted.stack; let services = &booted.services; let prepared = &booted.prepared; + let session_title = format!("Integration {} {}", prepared.scenario.id, runner.run_id); - let session_title = format!("Console E2E {} {}", prepared.scenario.id, runner.run_id); services .client() .call_with_deadline( @@ -204,67 +193,166 @@ async fn run_observe_phases( .map_err(|error| { RunError::setup( RunPhase::Arm, - "ensure console test session", + "ensure playground session", anyhow::anyhow!(error), ) })?; - let ready = build_ready_manifest(runner, prepared, stack, &session_title); - runner.write_run_artifact("observe-ready.json", &ready, RunPhase::Report)?; - write_atomic_json(ready_file, &ready).map_err(|error| { - RunError::runner(RunPhase::Report, "publish observe ready manifest", error) - })?; - - let start_file = start_file_path(ready_file)?; - wait_for_start(&start_file, prepared.setup_deadline).await?; + let console_url = stack + .spawn_console(console_bin) + .map_err(|error| RunError::setup(RunPhase::Arm, "spawn production Console", error))?; + wait_for_console(stack, &console_url, prepared.setup_deadline).await?; - let mut active = runner.send(services, prepared).await?; - runner.fault(stack, services, prepared, &active).await?; - runner.release(services, prepared, &active).await?; - runner.r#await(services, &mut active).await?; - - let scenario_deadline = active.deadline; - wait_for_shutdown(stack, scenario_deadline).await?; + let ready = build_ready_manifest(runner, prepared, stack, &session_title, &console_url); + runner.write_run_artifact("playground-ready.json", &ready, RunPhase::Report)?; + if let Some(path) = ready_file { + write_atomic_json(path, &ready).map_err(|error| { + RunError::runner(RunPhase::Report, "publish playground ready manifest", error) + })?; + } + println!("Console: {console_url}"); + std::io::stdout() + .flush() + .map_err(|error| RunError::runner(RunPhase::Report, "flush Console URL", error))?; + + let deadline = Deadline::after(Duration::from_millis( + prepared.scenario.deadlines.scenario_ms, + )); + let mut active = ActiveTurn::external(deadline); + let shutdown_consumed = wait_for_external_turn(runner, stack, services, &mut active).await?; + if !shutdown_consumed { + wait_for_shutdown(stack, deadline).await?; + } runner.collect(services, prepared, &mut active).await?; - let evidence = runner.build_evidence(services, &active, Some(active.send_response.clone())); + let evidence = runner.build_evidence(services, &active, None); runner.evidence = serde_json::to_value(&evidence).map_err(|error| { - RunError::runner(RunPhase::Grade, "serialize observe run evidence", error) + RunError::runner(RunPhase::Grade, "serialize playground run evidence", error) })?; runner.verify_evidence(services, &evidence, active.timed_out) } +async fn wait_for_external_turn( + runner: &mut ScenarioRunner<'_>, + stack: &mut Stack, + services: &crate::services::RunServices, + active: &mut ActiveTurn, +) -> Result { + let completion = runner.r#await(services, active); + let shutdown = shutdown_signal(); + tokio::pin!(completion, shutdown); + let mut health = tokio::time::interval(PROCESS_POLL_INTERVAL); + health.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + result = &mut completion => return result.map(|()| false), + result = &mut shutdown => { + result.map_err(|error| RunError::runner( + RunPhase::Await, + "wait for playground shutdown", + error, + ))?; + return match tokio::time::timeout( + SHUTDOWN_COMPLETION_GRACE, + &mut completion, + ) + .await + { + Ok(result) => result.map(|()| true), + Err(_) => Err(RunError::new( + RunPhase::Await, + RunErrorKind::Contract, + "playground stopped before a turn completed", + )), + }; + } + _ = health.tick() => { + if let Some(exit) = stack.early_exit() { + return Err(process_exit_error(exit, "before a turn completed")); + } + } + } + } +} + +async fn wait_for_console( + stack: &mut Stack, + console_url: &str, + deadline: Deadline, +) -> Result<(), RunError> { + let port = console_url + .rsplit(':') + .next() + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| { + RunError::new( + RunPhase::Arm, + RunErrorKind::Runner, + format!("invalid Console URL {console_url:?}"), + ) + })?; + loop { + if tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .is_ok() + { + return Ok(()); + } + if let Some(exit) = stack.early_exit() { + return Err(process_exit_error( + exit, + "before its HTTP port became ready", + )); + } + if deadline.is_expired() { + return Err(RunError::new( + RunPhase::Arm, + RunErrorKind::Setup, + "Console HTTP port did not become ready", + )); + } + tokio::time::sleep(PROCESS_POLL_INTERVAL).await; + } +} + fn build_ready_manifest( runner: &ScenarioRunner<'_>, prepared: &PreparedRun, stack: &Stack, session_title: &str, -) -> ObserveReadyV1 { + console_url: &str, +) -> PlaygroundReadyV1 { let prefix = format!("{}::", runner.run_id); - let functions = std::iter::once(&prepared.scenario.recorder.target) - .chain(prepared.scenario.recorder.extra_functions.iter()) - .filter_map(|function| { - function - .function_id - .strip_prefix(&prefix) - .map(|alias| (alias.to_string(), function.function_id.clone())) + let functions = prepared + .scenario + .recorder + .target + .function_id + .strip_prefix(&prefix) + .filter(|alias| *alias != "unused") + .map(|alias| { + BTreeMap::from([( + alias.to_string(), + prepared.scenario.recorder.target.function_id.clone(), + )]) }) - .collect(); + .unwrap_or_default(); let run_root = stack.paths.root.clone(); - ObserveReadyV1 { + PlaygroundReadyV1 { schema_version: SchemaVersion1::V1, run_id: runner.run_id.clone(), scenario_id: prepared.scenario.id.clone(), scenario_slug: runner.fixture.slug.clone(), driver: runner.fixture.driver, - result_path: run_root.join("observe-result.json"), + result_path: run_root.join("playground-result.json"), run_root, engine_url: stack.ws_url.clone(), - session: ObserveSessionV1 { + console_url: console_url.to_string(), + session: PlaygroundSessionV1 { id: runner.session_id.clone(), title: session_title.to_string(), }, - model: ObserveModelV1 { + model: PlaygroundModelV1 { id: prepared.scenario.send.model.clone(), provider: prepared.scenario.send.provider.clone(), }, @@ -274,53 +362,6 @@ fn build_ready_manifest( } } -fn start_file_path(ready_file: &Path) -> Result { - let parent = ready_file.parent().ok_or_else(|| { - RunError::new( - RunPhase::Probe, - RunErrorKind::Runner, - "ready file path has no parent directory for start.json", - ) - })?; - Ok(parent.join("start.json")) -} - -async fn wait_for_start(start_file: &Path, deadline: Deadline) -> Result<(), RunError> { - deadline - .poll_until("observer start signal", START_POLL_INTERVAL, || { - let start_file = start_file.to_path_buf(); - async move { - match std::fs::read_to_string(&start_file) { - Ok(contents) => { - let parsed: ObserveStartV1 = serde_json::from_str(&contents) - .map_err(|error| anyhow::anyhow!("invalid start.json: {error}"))?; - let _ = parsed; - Ok(Some(())) - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(error) => Err(anyhow::anyhow!( - "read start signal {}: {error}", - start_file.display() - )), - } - } - }) - .await - .map_err(|error| { - let kind = if deadline.is_expired() { - RunErrorKind::Timeout - } else { - RunErrorKind::Setup - }; - RunError::with_source( - RunPhase::Probe, - kind, - "observer did not publish start.json", - error, - ) - }) -} - async fn wait_for_shutdown(stack: &mut Stack, deadline: Deadline) -> Result<(), RunError> { let shutdown = shutdown_signal(); tokio::pin!(shutdown); @@ -330,7 +371,7 @@ async fn wait_for_shutdown(stack: &mut Stack, deadline: Deadline) -> Result<(), tokio::select! { result = &mut shutdown => { result.map_err(|error| { - RunError::runner(RunPhase::Await, "wait for observe test shutdown", error) + RunError::runner(RunPhase::Await, "wait for playground shutdown", error) })?; return Ok(()); } @@ -338,25 +379,26 @@ async fn wait_for_shutdown(stack: &mut Stack, deadline: Deadline) -> Result<(), return Err(RunError::new( RunPhase::Await, RunErrorKind::Timeout, - "observe test did not finish before the scenario deadline", + "playground was not stopped before the scenario deadline", )); } _ = health.tick() => { if let Some(exit) = stack.early_exit() { - return Err(RunError::new( - RunPhase::Await, - RunErrorKind::ProcessCrash, - format!( - "{} exited while the observe test was running: {}", - exit.name, exit.status - ), - )); + return Err(process_exit_error(exit, "while the playground was running")); } } } } } +fn process_exit_error(exit: crate::stack::EarlyExit, context: &str) -> RunError { + RunError::new( + RunPhase::Await, + RunErrorKind::ProcessCrash, + format!("{} exited {context}: {}", exit.name, exit.status), + ) +} + async fn shutdown_signal() -> std::io::Result<()> { #[cfg(unix)] { diff --git a/harness/evals/integration/src/scenario/runner.rs b/harness/evals/integration/src/scenario/runner.rs index 967ed91d5..c6f5907d3 100644 --- a/harness/evals/integration/src/scenario/runner.rs +++ b/harness/evals/integration/src/scenario/runner.rs @@ -132,11 +132,11 @@ pub(super) struct ScenarioRunner<'a> { /// First floor or verify failure, scrubbed of run-scoped ids. pub(super) failure: Option, /// Raw serialized [`crate::evidence_data::RunEvidence`], published in - /// observe results for Playwright; JSON null until collected. + /// playground results for Playwright; JSON null until collected. pub(super) evidence: serde_json::Value, } -/// Allocate + expand result shared by Direct and Observe drivers. +/// Allocate + expand result shared by Direct and Playground drivers. pub(super) struct ExpandedRun { pub paths: RunLayout, pub expanded: ExpandedFixtureV1, @@ -252,7 +252,7 @@ impl<'a> ScenarioRunner<'a> { }) } - /// Arm a booted stack (shared by Direct and Observe drivers). + /// Arm a booted stack (shared by Direct and Playground drivers). pub(super) async fn arm_booted(&mut self, booted: &mut BootedRun) -> Result<(), RunError> { self.arm(&mut booted.stack, &booted.services, &booted.prepared) .await @@ -270,7 +270,7 @@ impl<'a> ScenarioRunner<'a> { let outcome = async { self.arm_booted(&mut booted).await?; - self.run_phases_after_arm(&mut booted.stack, &booted.services, &booted.prepared) + self.run_phases_after_arm(&booted.services, &booted.prepared) .await } .await; @@ -285,13 +285,10 @@ impl<'a> ScenarioRunner<'a> { async fn run_phases_after_arm( &mut self, - stack: &mut Stack, services: &RunServices, prepared: &PreparedRun, ) -> Result<(), RunError> { let mut active = self.send(services, prepared).await?; - self.fault(stack, services, prepared, &active).await?; - self.release(services, prepared, &active).await?; self.r#await(services, &mut active).await?; self.collect(services, prepared, &mut active).await?; let evidence = self.build_evidence(services, &active, Some(active.send_response.clone())); @@ -317,7 +314,7 @@ impl<'a> ScenarioRunner<'a> { combine_teardown(classification, teardown_complete, artifact) } - /// Shared teardown tail for run and observe. Inspects process state while + /// Shared teardown tail for run and playground. Inspects process state while /// the subject is still running; service shutdown is intentionally /// before process teardown, and a second inspection catches a child /// that exits during that boundary. diff --git a/harness/evals/integration/src/scenario/state.rs b/harness/evals/integration/src/scenario/state.rs index e5596e212..eaf824b26 100644 --- a/harness/evals/integration/src/scenario/state.rs +++ b/harness/evals/integration/src/scenario/state.rs @@ -50,4 +50,8 @@ impl ActiveTurn { timed_out: false, } } + + pub(super) fn external(deadline: Deadline) -> Self { + Self::new(deadline, None, Value::Null) + } } diff --git a/harness/evals/integration/src/scenarios/builder.rs b/harness/evals/integration/src/scenarios/builder.rs deleted file mode 100644 index e7b012b11..000000000 --- a/harness/evals/integration/src/scenarios/builder.rs +++ /dev/null @@ -1,387 +0,0 @@ -//! Typed builders for authored scenarios. -//! -//! Builders produce data only: every method fills a field on the authored -//! structs in [`crate::types::scenario`] and returns the value for further -//! chaining. A builder that derives scenario content from control flow is -//! rejected in review, under the same rule that forbids a second -//! orchestration language in the authored layer. -//! -//! Defaults mirror the deterministic compiler defaults: a text reply without -//! chunks emits one terminal `done` frame, faults interrupt the first target -//! call after 1500 ms, and releases target `call-1`. - -use serde_json::{json, Value}; - -use crate::types::frames::Usage; -use crate::types::scenario::{ - AuthoredScenarioV1, FaultKind, FaultV1, GenerationMatchOverridesV1, ReleaseActionV1, ReleaseV1, - RouterReplyV1, ScenarioFunctionV1, ScenarioGenerationV1, ScenarioRouterV1, ScenarioSendV1, - TriggerBindingSpecV1, TriggerKindV1, -}; -use crate::types::script::JsonMatcherV1; - -/// Authoring name for the scenario root; [`Self::verify`] closes the chain. -pub type AuthoredScenario = AuthoredScenarioV1; - -/// A registration-ready scenario: the authored stimulus paired with its -/// checks. Produced by [`AuthoredScenarioV1::verify`], always the last -/// authoring step — a scenario without checks does not typecheck. -pub struct Scenario { - pub authored: AuthoredScenarioV1, - pub verify: super::VerifyFn, -} - -impl AuthoredScenarioV1 { - /// A new scenario with an empty send. Chain [`Self::trigger`] before - /// registering; the registry tests reject an empty message. - pub fn new(id: &str, description: &str) -> Self { - Self { - id: id.to_string(), - description: description.to_string(), - quarantine: false, - send: Harness::send(""), - functions: Default::default(), - router: ScenarioRouterV1 { - model: None, - generations: Vec::new(), - }, - bindings: Vec::new(), - release: None, - fault: None, - timeouts: Default::default(), - } - } - - /// Exclude this scenario from ordinary `all` runs; `validate` and - /// explicit selection still include it. - pub fn quarantine(mut self) -> Self { - self.quarantine = true; - self - } - - /// The turn-initiating invocation, mirroring the platform verb: the - /// compiled payload is what `iii.trigger("harness::send", …)` submits — - /// directly by the runner in Rpc mode, via the ready manifest by - /// Playwright in Serve mode. - pub fn trigger(mut self, send: ScenarioSendV1) -> Self { - self.send = send; - self - } - - /// Register a controlled function under its alias. The compiler expands - /// aliases to `{{run_id}}::`. - pub fn function(mut self, alias: &str, function: ScenarioFunctionV1) -> Self { - self.functions.insert(alias.to_string(), function); - self - } - - /// The scripted model conversation, one reply per generation. Accepts a - /// tuple so text and function-call replies can mix while keeping their - /// per-kind builders. - pub fn model(mut self, generations: impl IntoGenerations) -> Self { - self.router.generations = generations.into_generations(); - self - } - - pub fn binding(mut self, binding: TriggerBindingSpecV1) -> Self { - self.bindings.push(binding); - self - } - - /// Release the deterministic first call (`call-1`) once it is held. - pub fn release(mut self, action: ReleaseActionV1) -> Self { - self.release = Some(ReleaseV1 { - function_call_id: "call-1".to_string(), - action, - }); - self - } - - pub fn fault(mut self, fault: FaultV1) -> Self { - self.fault = Some(fault); - self - } - - /// The scenario's checks over the returned - /// [`crate::evidence_data::RunEvidence`] dataset; the runner enforces the - /// floor (turn completed, script fully consumed, clean send) before this - /// runs. Always the last builder step. - pub fn verify(self, verify: super::VerifyFn) -> Scenario { - Scenario { - authored: self, - verify, - } - } - - pub fn scenario_timeout_ms(mut self, scenario_ms: u64) -> Self { - self.timeouts.scenario_ms = scenario_ms; - self - } -} - -/// Entry point for [`ScenarioSendV1`]. `Harness::send(…)` spells the wire -/// function id `harness::send`, so `.trigger(Harness::send(…))` reads like -/// the platform call `iii.trigger("harness::send", …)`. -pub struct Harness; - -impl Harness { - pub fn send(message: &str) -> ScenarioSendV1 { - ScenarioSendV1 { - message: message.to_string(), - } - } -} - -/// Authoring name for [`ScenarioFunctionV1`]. -pub type Function = ScenarioFunctionV1; - -impl ScenarioFunctionV1 { - /// A controlled function exposed to the model. `request_schema` must be - /// a JSON object. - pub fn new(description: &str, request_schema: Value, response: Value) -> Self { - Self { - description: description.to_string(), - request_schema: request_schema - .as_object() - .cloned() - .expect("request_schema must be a JSON object"), - response, - expose: true, - } - } - - /// The canonical recorder fixture: one required string `value`, returning - /// a durable `recorded` text result. It is the most common controlled - /// function; chain [`Self::hidden`] for a hook-only variant. - pub fn recorder() -> Self { - Self::new( - "Record one integration fixture value.", - json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }), - json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - ) - } - - /// Hook-only controlled functions are never exposed to the model. - pub fn hidden(mut self) -> Self { - self.expose = false; - self - } -} - -/// Entry point for typed router replies. -pub struct Reply; - -impl Reply { - pub fn text(text: &str) -> TextReply { - TextReply { - text: text.to_string(), - chunks: Vec::new(), - usage: None, - } - } - - /// A function call against a registered alias, closed as - /// `call-` by the compiler. - pub fn function_call(function: &str, arguments: Value) -> FunctionCallReply { - FunctionCallReply { - function: function.to_string(), - arguments, - usage: None, - } - } -} - -pub struct TextReply { - text: String, - chunks: Vec, - usage: Option, -} - -impl TextReply { - /// Non-empty chunks produce the complete streaming frame sequence; - /// without chunks the reply is one terminal `done` frame. - pub fn chunks(mut self, chunks: [&str; N]) -> Self { - self.chunks = chunks.iter().map(|chunk| chunk.to_string()).collect(); - self - } - - pub fn usage(mut self, input: u64, output: u64) -> Self { - self.usage = Some(input_output_usage(input, output)); - self - } - - pub fn match_overrides(self, overrides: GenerationMatchOverridesV1) -> ScenarioGenerationV1 { - with_overrides(self.into(), overrides) - } - - /// Match this reply against the durable outcome only, at a recovery - /// boundary (fault restart or hook release) where the engine may rebuild - /// the request differently. Shorthand for [`Self::match_overrides`] with - /// the recovery policy. - pub fn recovery_boundary(self) -> ScenarioGenerationV1 { - with_overrides(self.into(), recovery_overrides()) - } -} - -impl From for ScenarioGenerationV1 { - fn from(reply: TextReply) -> Self { - plain_generation(RouterReplyV1::Text { - text: reply.text, - chunks: reply.chunks, - usage: reply.usage, - }) - } -} - -pub struct FunctionCallReply { - function: String, - arguments: Value, - usage: Option, -} - -impl FunctionCallReply { - pub fn usage(mut self, input: u64, output: u64) -> Self { - self.usage = Some(input_output_usage(input, output)); - self - } -} - -impl From for ScenarioGenerationV1 { - fn from(reply: FunctionCallReply) -> Self { - plain_generation(RouterReplyV1::FunctionCall { - id: None, - function: reply.function, - arguments: reply.arguments, - usage: reply.usage, - }) - } -} - -fn plain_generation(reply: RouterReplyV1) -> ScenarioGenerationV1 { - ScenarioGenerationV1 { - reply, - match_overrides: Default::default(), - } -} - -fn with_overrides( - mut generation: ScenarioGenerationV1, - overrides: GenerationMatchOverridesV1, -) -> ScenarioGenerationV1 { - generation.match_overrides = overrides; - generation -} - -/// The loose match a recovery-boundary reply needs. After a fault restart or a -/// hook release the engine may legitimately reconstruct the request -/// differently, so match the durable shape and leave the reconstructed request -/// id and body free. -fn recovery_overrides() -> GenerationMatchOverridesV1 { - GenerationMatchOverridesV1 { - request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), - system_prompt: Some(present()), - messages: Some(present()), - tools: Some(present()), - } -} - -fn input_output_usage(input: u64, output: u64) -> Usage { - Usage { - input: Some(input), - output: Some(output), - cache_read: None, - cache_write: None, - reasoning: None, - cost_usd: None, - } -} - -/// Entry point for [`TriggerBindingSpecV1`]. -pub struct Binding; - -impl Binding { - pub fn hook_pre_trigger( - function: &str, - functions: [&str; N], - priority: i64, - ) -> TriggerBindingSpecV1 { - TriggerBindingSpecV1 { - trigger: TriggerKindV1::HookPreTrigger, - function: function.to_string(), - functions: functions.iter().map(|alias| alias.to_string()).collect(), - priority, - } - } -} - -/// Entry point for [`FaultV1`] with deterministic defaults. -pub struct Fault; - -impl Fault { - pub fn engine_sigkill() -> FaultV1 { - FaultV1 { - kind: FaultKind::EngineSigkill, - after_target_calls: 1, - restart_delay_ms: 1_500, - } - } -} - -/// Entry point for [`ReleaseActionV1`], the action applied to the held -/// deterministic first call by [`AuthoredScenarioV1::release`]. -pub struct Release; - -impl Release { - /// Release the held call for execution against its target function. - pub fn execute() -> ReleaseActionV1 { - ReleaseActionV1::Execute - } -} - -/// Tuple-to-generations conversion for [`AuthoredScenarioV1::model`], so a -/// scripted conversation can mix reply kinds without erasing their types. -pub trait IntoGenerations { - fn into_generations(self) -> Vec; -} - -macro_rules! impl_into_generations { - ($(($($name:ident),+);)+) => {$( - #[allow(non_snake_case)] - impl<$($name: Into),+> IntoGenerations for ($($name,)+) { - fn into_generations(self) -> Vec { - let ($($name,)+) = self; - vec![$($name.into()),+] - } - } - )+}; -} - -impl_into_generations! { - (G1); - (G1, G2); -} - -pub fn regex(pattern: &str) -> JsonMatcherV1 { - JsonMatcherV1::Regex { - pattern: pattern.to_string(), - } -} - -pub fn present() -> JsonMatcherV1 { - JsonMatcherV1::Present -} - -pub fn subset(expected: Value) -> JsonMatcherV1 { - JsonMatcherV1::Subset { - expected, - normalize: None, - } -} diff --git a/harness/evals/integration/src/scenarios/console_streamed_text.rs b/harness/evals/integration/src/scenarios/console_streamed_text.rs index e618447e6..341f2a460 100644 --- a/harness/evals/integration/src/scenarios/console_streamed_text.rs +++ b/harness/evals/integration/src/scenarios/console_streamed_text.rs @@ -1,27 +1,77 @@ -//! UI-001 — integration starts a streamed turn; Playwright validates Console UI. +//! UI-001 — Playwright sends a Console turn and validates the rendered result. use anyhow::ensure; +use serde_json::json; -use super::builder::*; +use super::support::{ + model, request_match, response, send, streamed_text_frames, synthetic_recorder, system_prompt, + usage, user_message, RequestProfile, +}; +use super::ScenarioDriver; +use crate::fixtures::ScenarioFixture; +use crate::types::frames::StopReason; +use crate::types::scenario::{CompiledScenarioV1, DeadlinesV1}; +use crate::types::script::{RouterScriptV1, SchemaVersion1, ScriptedGenerationV1}; -pub(super) fn scenario() -> Scenario { - AuthoredScenario::new( - "UI-001", - "A harness-started streamed turn renders to durable completion in the Console.", - ) - .trigger(Harness::send("Return the console fixture phrase.")) - .model((Reply::text("console fixture complete") - .chunks(["console fixture ", "complete"]) - .usage(9, 3),)) - // Content assertions for UI scenarios live in Playwright (`ui-send` checks - // the rendered text and both message counts in the DOM); the floor (turn - // completion, script consumption, clean send) is runner-owned, so this - // only checks what the DOM cannot show. - .verify(|run| { - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" - ); - Ok(()) - }) +pub(super) fn scenario() -> ScenarioFixture { + const ID: &str = "UI-001"; + const MESSAGE: &str = "Return the console fixture phrase."; + const TEXT: &str = "console fixture complete"; + + let model = model(); + let usage = usage(9, 3); + let allowed_functions = Vec::new(); + let messages = vec![user_message(MESSAGE)]; + let generation = ScriptedGenerationV1 { + ordinal: 1, + match_: request_match(1, &model, &messages, &json!([]), RequestProfile::Console), + frames: streamed_text_frames(TEXT, &["console fixture ", "complete"], &usage, &model), + response: response(StopReason::End, usage, &model), + }; + + ScenarioFixture { + slug: "console-streamed-text".to_string(), + driver: ScenarioDriver::Playground, + scenario: CompiledScenarioV1 { + schema_version: SchemaVersion1::V1, + id: ID.to_string(), + description: "A Console-sent streamed turn reaches durable completion.".to_string(), + send: send(ID, MESSAGE, &model, &allowed_functions), + recorder: synthetic_recorder(), + deadlines: DeadlinesV1::default(), + }, + script: RouterScriptV1 { + schema_version: SchemaVersion1::V1, + scenario_id: ID.to_string(), + model, + generations: vec![generation], + }, + system_prompt_template: system_prompt(&allowed_functions), + verify: |run| { + ensure!( + !run.has_duplicate_messages(), + "transcript contains duplicate entry ids" + ); + Ok(()) + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::script::JsonMatcherV1; + + #[test] + fn accepts_console_prompt_and_tool_shape() { + let fixture = scenario(); + assert!(matches!( + fixture.script.generations[0].match_.system_prompt, + JsonMatcherV1::Regex { .. } + )); + assert!(matches!( + fixture.script.generations[0].match_.tools, + JsonMatcherV1::Subset { .. } + )); + } } diff --git a/harness/evals/integration/src/scenarios/crash_recovery_507.rs b/harness/evals/integration/src/scenarios/crash_recovery_507.rs deleted file mode 100644 index 2ab923f27..000000000 --- a/harness/evals/integration/src/scenarios/crash_recovery_507.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! E2E-507 — crash recovery closes the interrupted function call. -//! -//! Reproduction of . - -use anyhow::ensure; -use serde_json::json; - -use super::builder::*; - -pub(super) fn scenario() -> Scenario { - AuthoredScenario::new( - "E2E-507", - "An engine crash during a dispatched function call must not leave the call dangling or the session unusable.", - ) - .quarantine() - .trigger(Harness::send("Call the recorder once.")) - .function("record", Function::recorder()) - .model(( - Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4), - // Recovery can legitimately reconstruct the second request - // differently; this reproduction checks the durable outcome instead. - Reply::text("recovered").usage(20, 2).recovery_boundary(), - )) - .fault(Fault::engine_sigkill()) - .scenario_timeout_ms(120_000) - // Recovery is proven by the closed call, the message shape, and both - // generations being consumed (floor); text durability is `streamed-text`'s - // pin. -.verify(|run| { - let counts = run.message_counts(); - ensure!( - counts == (1, 2, 1), - "message counts (user, assistant, function_result) {counts:?} != (1, 2, 1)" - ); - ensure!( - run.all_calls_closed(), - "a dispatched function call has no durable result" - ); - ensure!( - run.function_result_closes("call-1"), - "no single durable function result closes call-1" - ); - let calls = run.calls("record"); - ensure!( - calls.len() == 1, - "record ran {} times, not exactly once", - calls.len() - ); - let payload = &calls[0].payload; - ensure!( - payload == &json!({ "value": "expected" }), - "record payload {payload} != {{\"value\":\"expected\"}}" - ); - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" - ); - Ok(()) - }) -} diff --git a/harness/evals/integration/src/scenarios/exactly_once_function.rs b/harness/evals/integration/src/scenarios/exactly_once_function.rs index e19af3c13..2ac4476e7 100644 --- a/harness/evals/integration/src/scenarios/exactly_once_function.rs +++ b/harness/evals/integration/src/scenarios/exactly_once_function.rs @@ -3,17 +3,116 @@ use anyhow::ensure; use serde_json::json; -use super::builder::*; - -pub(super) fn scenario() -> Scenario { - AuthoredScenario::new("E2E-002", "The recorder runs exactly once.") - .trigger(Harness::send("Call the recorder once.")) - .function("record", Function::recorder()) - .model(( - Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4), - Reply::text("recorded once").usage(18, 2), - )) - .verify(|run| { +use super::support::{ + assistant_message, model, recorder, recorder_target, request_match, response, send, + system_prompt, usage, user_message, RequestProfile, MODEL_ID, PROVIDER_ID, +}; +use super::ScenarioDriver; +use crate::fixtures::ScenarioFixture; +use crate::types::frames::{AssistantMessageEvent, ContentBlock, StopReason}; +use crate::types::scenario::{CompiledScenarioV1, DeadlinesV1}; +use crate::types::script::{RouterScriptV1, SchemaVersion1, ScriptedGenerationV1}; + +pub(super) fn scenario() -> ScenarioFixture { + const ID: &str = "E2E-002"; + const MESSAGE: &str = "Call the recorder once."; + const FUNCTION_ID: &str = "{{run_id}}::record"; + + let model = model(); + let target = recorder_target(FUNCTION_ID); + let allowed_functions = vec![FUNCTION_ID.to_string()]; + let tools = json!([{ + "name": FUNCTION_ID, + "description": target.description, + "parameters": target.request_schema, + "execution_mode": "sequential" + }]); + let arguments = json!({ "value": "expected" }); + let call_usage = usage(8, 4); + let final_usage = usage(18, 2); + + let first_messages = vec![user_message(MESSAGE)]; + let function_call = assistant_message( + vec![ContentBlock::FunctionCall { + id: "call-1".to_string(), + function_id: FUNCTION_ID.to_string(), + arguments: arguments.clone(), + }], + StopReason::FunctionCall, + Some(call_usage.clone()), + &model, + 1, + ); + let mut second_messages = first_messages.clone(); + second_messages.extend([ + json!({ + "role": "assistant", + "content": [{ + "type": "function_call", + "id": "call-1", + "function_id": FUNCTION_ID, + "arguments": arguments + }], + "stop_reason": "end", + "model": MODEL_ID, + "provider": PROVIDER_ID + }), + json!({ + "role": "function_result", + "function_call_id": "call-1", + "function_id": FUNCTION_ID, + "content": [{ "type": "text", "text": "recorded" }], + "details": target.response, + "is_error": false + }), + ]); + let final_message = assistant_message( + vec![ContentBlock::Text { + text: "recorded once".to_string(), + }], + StopReason::End, + Some(final_usage.clone()), + &model, + 2, + ); + let generations = vec![ + ScriptedGenerationV1 { + ordinal: 1, + match_: request_match(1, &model, &first_messages, &tools, RequestProfile::Direct), + frames: vec![AssistantMessageEvent::Done { + message: function_call, + }], + response: response(StopReason::FunctionCall, call_usage, &model), + }, + ScriptedGenerationV1 { + ordinal: 2, + match_: request_match(2, &model, &second_messages, &tools, RequestProfile::Direct), + frames: vec![AssistantMessageEvent::Done { + message: final_message, + }], + response: response(StopReason::End, final_usage, &model), + }, + ]; + + ScenarioFixture { + slug: "exactly-once-function".to_string(), + driver: ScenarioDriver::Direct, + scenario: CompiledScenarioV1 { + schema_version: SchemaVersion1::V1, + id: ID.to_string(), + description: "The recorder runs exactly once.".to_string(), + send: send(ID, MESSAGE, &model, &allowed_functions), + recorder: recorder(target), + deadlines: DeadlinesV1::default(), + }, + script: RouterScriptV1 { + schema_version: SchemaVersion1::V1, + scenario_id: ID.to_string(), + model, + generations, + }, + system_prompt_template: system_prompt(&allowed_functions), + verify: |run| { let texts = run.assistant_texts(); ensure!( texts == ["recorded once"], @@ -35,5 +134,24 @@ pub(super) fn scenario() -> Scenario { "transcript contains duplicate entry ids" ); Ok(()) - }) + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::script::JsonMatcherV1; + + #[test] + fn pins_function_call_and_result_history() { + let fixture = scenario(); + let JsonMatcherV1::Exact { expected, .. } = &fixture.script.generations[1].match_.messages + else { + panic!("function history must be exact") + }; + assert_eq!(expected.as_array().unwrap().len(), 3); + assert_eq!(expected[1]["content"][0]["id"], "call-1"); + assert_eq!(expected[2]["role"], "function_result"); + } } diff --git a/harness/evals/integration/src/scenarios/hold_mutation_505.rs b/harness/evals/integration/src/scenarios/hold_mutation_505.rs deleted file mode 100644 index 04711a862..000000000 --- a/harness/evals/integration/src/scenarios/hold_mutation_505.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! E2E-505 — a holding hook's mutation reaches the released call. -//! -//! Reproduction of . - -use anyhow::ensure; -use serde_json::json; - -use crate::evidence_data::json_contains; - -use super::builder::*; - -pub(super) fn scenario() -> Scenario { - AuthoredScenario::new( - "E2E-505", - "A pre-trigger hook that holds and mutates must apply its mutation to the released call.", - ) - .quarantine() - .trigger(Harness::send("Call the recorder once.")) - .function("record", Function::recorder()) - .function( - "hook-gate", - Function::new( - "Hold the call and stamp approval context onto its arguments.", - json!({ "type": "object" }), - json!({ - "decision": "hold", - "mutations": { "arguments": { "value": "expected+approved" } } - }), - ) - .hidden(), - ) - .binding(Binding::hook_pre_trigger("hook-gate", ["record"], 10)) - .release(Release::execute()) - .model(( - Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4), - Reply::text("approved and recorded") - .usage(20, 3) - .recovery_boundary(), - )) - .verify(|run| { - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" - ); - ensure!( - run.all_calls_closed(), - "a dispatched function call has no durable result" - ); - - let record = run.calls("record"); - ensure!( - record.len() == 1, - "record ran {} times, not exactly once", - record.len() - ); - let payload = &record[0].payload; - ensure!( - payload == &json!({ "value": "expected+approved" }), - "record payload {payload} lost the hook mutation" - ); - - let gate = run.calls("hook-gate"); - ensure!( - gate.len() == 1, - "hook-gate ran {} times, not exactly once", - gate.len() - ); - // Hook payloads carry engine-populated fields beyond this subset. - let consulted = json!({ - "point": "pre_trigger", - "call": { - "id": "call-1", - "function_id": format!("{}::record", run.run_id), - "arguments": { "value": "expected" } - } - }); - ensure!( - json_contains(&gate[0].payload, &consulted), - "hook-gate payload {} does not contain {consulted}", - gate[0].payload - ); - Ok(()) - }) -} diff --git a/harness/evals/integration/src/scenarios/hook_held_release_506.rs b/harness/evals/integration/src/scenarios/hook_held_release_506.rs deleted file mode 100644 index b9fdd47a3..000000000 --- a/harness/evals/integration/src/scenarios/hook_held_release_506.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! E2E-506 — released held calls retain hook-mutated arguments. -//! -//! Reproduction of . - -use anyhow::ensure; -use serde_json::json; - -use crate::evidence_data::json_contains; - -use super::builder::*; - -pub(super) fn scenario() -> Scenario { - AuthoredScenario::new( - "E2E-506", - "A held call released for execution must run with the arguments produced by earlier hooks.", - ) - .quarantine() - .trigger(Harness::send("Call the recorder once.")) - .function("record", Function::recorder()) - .function( - "hook-mutate", - Function::new( - "Inject validated scope into the arguments.", - json!({ "type": "object" }), - json!({ - "decision": "continue", - "mutations": { "arguments": { "value": "expected+scope" } } - }), - ) - .hidden(), - ) - .function( - "hook-hold", - Function::new( - "Hold every consulted call for explicit approval.", - json!({ "type": "object" }), - json!({ "decision": "hold" }), - ) - .hidden(), - ) - .binding(Binding::hook_pre_trigger("hook-mutate", ["record"], 10)) - .binding(Binding::hook_pre_trigger("hook-hold", ["record"], 20)) - .release(Release::execute()) - .model(( - Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4), - Reply::text("released and recorded") - .usage(20, 3) - .recovery_boundary(), - )) - .verify(|run| { - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" - ); - ensure!( - run.all_calls_closed(), - "a dispatched function call has no durable result" - ); - - let record = run.calls("record"); - ensure!( - record.len() == 1, - "record ran {} times, not exactly once", - record.len() - ); - let payload = &record[0].payload; - ensure!( - payload == &json!({ "value": "expected+scope" }), - "record payload {payload} lost the hook mutation" - ); - - // Hook payloads carry engine-populated fields beyond these subsets: - // hook-mutate is consulted with the original arguments, hook-hold with - // the arguments hook-mutate produced. - let hook_subset = |arguments: &str| { - json!({ - "point": "pre_trigger", - "call": { - "id": "call-1", - "function_id": format!("{}::record", run.run_id), - "arguments": { "value": arguments } - } - }) - }; - for (alias, arguments) in [("hook-mutate", "expected"), ("hook-hold", "expected+scope")] { - let calls = run.calls(alias); - ensure!( - calls.len() == 1, - "{alias} ran {} times, not exactly once", - calls.len() - ); - let consulted = hook_subset(arguments); - ensure!( - json_contains(&calls[0].payload, &consulted), - "{alias} payload {} does not contain {consulted}", - calls[0].payload - ); - } - Ok(()) - }) -} diff --git a/harness/evals/integration/src/scenarios/mod.rs b/harness/evals/integration/src/scenarios/mod.rs index c288aacc2..cea35fd72 100644 --- a/harness/evals/integration/src/scenarios/mod.rs +++ b/harness/evals/integration/src/scenarios/mod.rs @@ -1,128 +1,45 @@ -//! Authored scenario modules and their registry. -//! -//! One Rust module per scenario: each `src/scenarios/.rs` builds the -//! authored scenario data through the typed builders in [`builder`], defines -//! a `verify` function over the returned [`RunEvidence`] dataset, and -//! registers both in [`all`]. There is no YAML layer — the authored shape is -//! the runner's own data model, enforced at `cargo build`, and it is never -//! serialized. Compilation and serde round trips are enforced by tests -//! without checking in a second copy of each scenario. - -pub mod builder; +//! The three checked-in integration fixtures. mod console_streamed_text; -mod crash_recovery_507; mod exactly_once_function; -mod hold_mutation_505; -mod hook_held_release_506; mod streamed_text; +mod support; use crate::evidence_data::RunEvidence; -use crate::types::scenario::AuthoredScenarioV1; +use crate::fixtures::ScenarioFixture; -/// Scenario-specific checks written in plain Rust over the collected run -/// dataset. The runner enforces the floor (completed turn, script fully -/// consumed, clean send) before calling this, and catches panics, so -/// `assert!`/`assert_eq!` are allowed; prefer `anyhow::ensure!` where a -/// message helps. pub type VerifyFn = fn(&RunEvidence) -> anyhow::Result<()>; #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] pub enum ScenarioDriver { Direct, - Observe, -} - -/// One authored scenario, its verify function, and the stable slug used by -/// `--scenario` selection. -#[derive(Debug, Clone)] -pub struct RegisteredScenario { - pub slug: String, - pub authored: AuthoredScenarioV1, - pub driver: ScenarioDriver, - pub verify: VerifyFn, + Playground, } -/// Every authored scenario, in stable slug order. -pub fn all() -> Vec { +/// Every fixture, in stable slug order. +pub fn all() -> Vec { vec![ - register("crash-recovery-507", crash_recovery_507::scenario()), - register_observe("console-streamed-text", console_streamed_text::scenario()), - register("exactly-once-function", exactly_once_function::scenario()), - register("hold-mutation-505", hold_mutation_505::scenario()), - register("hook-held-release-506", hook_held_release_506::scenario()), - register("streamed-text", streamed_text::scenario()), + console_streamed_text::scenario(), + exactly_once_function::scenario(), + streamed_text::scenario(), ] } -fn register(slug: &str, scenario: builder::Scenario) -> RegisteredScenario { - RegisteredScenario { - slug: slug.to_string(), - authored: scenario.authored, - driver: ScenarioDriver::Direct, - verify: scenario.verify, - } -} - -fn register_observe(slug: &str, scenario: builder::Scenario) -> RegisteredScenario { - RegisteredScenario { - slug: slug.to_string(), - authored: scenario.authored, - driver: ScenarioDriver::Observe, - verify: scenario.verify, - } -} - #[cfg(test)] mod tests { use super::*; - use crate::expand::compile_scenario; - use crate::types::scenario::validate_scenario_id; - - /// Slugs are stable `--scenario` selectors and filesystem-safe labels. - fn validate_slug(slug: &str) { - assert!( - !slug.is_empty() - && slug - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')), - "slug {slug:?} must contain only ASCII letters, digits, '-' or '_'" - ); - } #[test] - fn every_scenario_registers_exactly_once() { - let registered = all(); - assert!(!registered.is_empty()); + fn every_fixture_is_unique_and_valid() { + let fixtures = all(); + assert_eq!(fixtures.len(), 3); let mut slugs = std::collections::BTreeSet::new(); let mut ids = std::collections::BTreeSet::new(); - for entry in ®istered { - validate_slug(&entry.slug); - validate_scenario_id(&entry.authored.id).unwrap(); - assert!( - slugs.insert(entry.slug.clone()), - "slug {:?} registered more than once", - entry.slug - ); - assert!( - ids.insert(entry.authored.id.clone()), - "scenario id {:?} registered more than once", - entry.authored.id - ); - } - } - - #[test] - fn every_scenario_compiles() { - for entry in all() { - assert!( - !entry.authored.send.message.is_empty(), - "{}: authored send message must not be empty", - entry.slug - ); - compile_scenario(&entry.authored, "base prompt\n") - .unwrap_or_else(|error| panic!("{} does not compile: {error:#}", entry.slug)); + for fixture in fixtures { + fixture.validate().unwrap(); + assert!(slugs.insert(fixture.slug)); + assert!(ids.insert(fixture.scenario.id)); } } } diff --git a/harness/evals/integration/src/scenarios/streamed_text.rs b/harness/evals/integration/src/scenarios/streamed_text.rs index c6d7e735d..9c3403f19 100644 --- a/harness/evals/integration/src/scenarios/streamed_text.rs +++ b/harness/evals/integration/src/scenarios/streamed_text.rs @@ -1,34 +1,88 @@ //! E2E-001 — streamed text reaches durable completion. use anyhow::ensure; +use serde_json::json; -use super::builder::*; +use super::support::{ + model, request_match, response, send, streamed_text_frames, synthetic_recorder, system_prompt, + usage, user_message, RequestProfile, +}; +use super::ScenarioDriver; +use crate::fixtures::ScenarioFixture; +use crate::types::frames::StopReason; +use crate::types::scenario::{CompiledScenarioV1, DeadlinesV1}; +use crate::types::script::{RouterScriptV1, SchemaVersion1, ScriptedGenerationV1}; -pub(super) fn scenario() -> Scenario { - AuthoredScenario::new( - "E2E-001", - "Streamed text reaches durable completion through the real queue and turn loop.", - ) - .trigger(Harness::send("Return the fixture phrase.")) - .model((Reply::text("fixture complete") - .chunks(["fixture ", "complete"]) - .usage(8, 2),)) - // The one direct pin for assistant-text durability and transcript shape. - .verify(|run| { - let texts = run.assistant_texts(); - ensure!( - texts == ["fixture complete"], - "assistant texts {texts:?} != [\"fixture complete\"]" - ); - let counts = run.message_counts(); - ensure!( - counts == (1, 1, 0), - "message counts (user, assistant, function_result) {counts:?} != (1, 1, 0)" - ); - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" +pub(super) fn scenario() -> ScenarioFixture { + const ID: &str = "E2E-001"; + const MESSAGE: &str = "Return the fixture phrase."; + const TEXT: &str = "fixture complete"; + + let model = model(); + let usage = usage(8, 2); + let allowed_functions = Vec::new(); + let messages = vec![user_message(MESSAGE)]; + let generation = ScriptedGenerationV1 { + ordinal: 1, + match_: request_match(1, &model, &messages, &json!([]), RequestProfile::Direct), + frames: streamed_text_frames(TEXT, &["fixture ", "complete"], &usage, &model), + response: response(StopReason::End, usage, &model), + }; + + ScenarioFixture { + slug: "streamed-text".to_string(), + driver: ScenarioDriver::Direct, + scenario: CompiledScenarioV1 { + schema_version: SchemaVersion1::V1, + id: ID.to_string(), + description: + "Streamed text reaches durable completion through the real queue and turn loop." + .to_string(), + send: send(ID, MESSAGE, &model, &allowed_functions), + recorder: synthetic_recorder(), + deadlines: DeadlinesV1::default(), + }, + script: RouterScriptV1 { + schema_version: SchemaVersion1::V1, + scenario_id: ID.to_string(), + model, + generations: vec![generation], + }, + system_prompt_template: system_prompt(&allowed_functions), + verify: |run| { + let texts = run.assistant_texts(); + ensure!(texts == [TEXT], "assistant texts {texts:?} != [\"{TEXT}\"]"); + let counts = run.message_counts(); + ensure!( + counts == (1, 1, 0), + "message counts (user, assistant, function_result) {counts:?} != (1, 1, 0)" + ); + ensure!( + !run.has_duplicate_messages(), + "transcript contains duplicate entry ids" + ); + Ok(()) + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stream_has_one_terminal_frame_and_matching_response() { + let fixture = scenario(); + let generation = &fixture.script.generations[0]; + assert_eq!( + generation + .frames + .iter() + .filter(|frame| frame.is_terminal()) + .count(), + 1 ); - Ok(()) - }) + assert!(generation.frames.last().unwrap().is_terminal()); + assert_eq!(generation.response.stop_reason, Some(StopReason::End)); + } } diff --git a/harness/evals/integration/src/scenarios/support.rs b/harness/evals/integration/src/scenarios/support.rs new file mode 100644 index 000000000..0b1347384 --- /dev/null +++ b/harness/evals/integration/src/scenarios/support.rs @@ -0,0 +1,314 @@ +//! Shared wire constructors used by the checked-in scenarios. + +use serde_json::{json, Value}; + +use crate::types::frames::{ + AssistantMessage, AssistantMessageEvent, AssistantRoleTag, ContentBlock, RouterChatResponse, + StopReason, Usage, +}; +use crate::types::recorder::{ + LifecycleFunctionId, LifecycleTriggerType, RecorderConfigV1, RecorderLifecycleV1, + RecorderTargetV1, +}; +use crate::types::scenario::{ + CompiledFunctionExposureV1, CompiledFunctionPolicyV1, CompiledSendOptionsV1, CompiledSendV1, +}; +use crate::types::script::{ + GenerationMatchV1, JsonMatcherV1, JsonNormalizerV1, ModelFixtureV1, NormalizerOperation, +}; + +const DEFAULT_SYSTEM_PROMPT: &str = include_str!("../../../../prompts/default.txt"); +pub(super) const MODEL_ID: &str = "fixture-model"; +pub(super) const PROVIDER_ID: &str = "scripted"; + +#[derive(Debug, Clone, Copy)] +pub(super) enum RequestProfile { + Direct, + Console, +} + +pub(super) fn model() -> ModelFixtureV1 { + ModelFixtureV1 { + id: MODEL_ID.to_string(), + provider: PROVIDER_ID.to_string(), + context_window: 32_768, + max_output_tokens: 4_096, + supports_thinking: Some(false), + supports_xhigh: None, + supports_tools: Some(true), + supports_vision: Some(false), + supports_cache: Some(false), + supports_structured_output: Some(true), + } +} + +pub(super) fn send( + scenario_id: &str, + message: &str, + model: &ModelFixtureV1, + allowed_functions: &[String], +) -> CompiledSendV1 { + CompiledSendV1 { + session_id: "{{session_id}}".to_string(), + message: message.to_string(), + model: model.id.clone(), + provider: model.provider.clone(), + idempotency_key: format!("{{{{run_id}}}}:{}", scenario_id.to_ascii_lowercase()), + options: CompiledSendOptionsV1 { + functions: CompiledFunctionPolicyV1 { + allow: allowed_functions.to_vec(), + deny: Vec::new(), + expose: CompiledFunctionExposureV1::Native, + }, + }, + } +} + +pub(super) fn request_match( + ordinal: u64, + model: &ModelFixtureV1, + messages: &[Value], + tools: &Value, + profile: RequestProfile, +) -> GenerationMatchV1 { + let normalizers = (0..messages.len()) + .map(|index| JsonNormalizerV1 { + pointer: format!("/{index}/timestamp"), + operation: NormalizerOperation::Delete, + }) + .collect(); + let (system_prompt, tools) = match profile { + RequestProfile::Direct => ( + JsonMatcherV1::Sha256 { + expected: "{{system_prompt_sha256}}".to_string(), + }, + exact(tools.clone()), + ), + RequestProfile::Console => ( + JsonMatcherV1::Regex { + pattern: "agent_trigger".to_string(), + }, + JsonMatcherV1::Subset { + expected: json!([{ "name": "agent_trigger" }]), + normalize: None, + }, + ), + }; + GenerationMatchV1 { + writer_ref: JsonMatcherV1::Subset { + expected: json!({ "direction": "write" }), + normalize: None, + }, + request_id: JsonMatcherV1::Regex { + pattern: if ordinal == 1 { + "^t_[0-9a-f]{32}:[0-9]+$".to_string() + } else { + format!("^t_[0-9a-f]{{32}}:{}$", ordinal - 1) + }, + }, + model: exact(json!(model.id)), + provider: exact(json!(model.provider)), + system_prompt, + messages: JsonMatcherV1::Exact { + expected: Value::Array(messages.to_vec()), + normalize: Some(normalizers), + }, + tools, + response_format: JsonMatcherV1::Absent, + thinking_level: JsonMatcherV1::Absent, + max_output_tokens: JsonMatcherV1::Absent, + provider_options: JsonMatcherV1::Absent, + metadata: JsonMatcherV1::Absent, + } +} + +fn exact(expected: Value) -> JsonMatcherV1 { + JsonMatcherV1::Exact { + expected, + normalize: None, + } +} + +pub(super) fn user_message(message: &str) -> Value { + json!({ + "role": "user", + "content": [{ "type": "text", "text": message }] + }) +} + +pub(super) fn assistant_message( + content: Vec, + stop_reason: StopReason, + usage: Option, + model: &ModelFixtureV1, + timestamp: i64, +) -> AssistantMessage { + AssistantMessage { + role: AssistantRoleTag::Assistant, + content, + stop_reason, + native_stop_reason: None, + error_message: None, + error_kind: None, + warnings: None, + usage, + model: model.id.clone(), + provider: model.provider.clone(), + timestamp, + } +} + +pub(super) fn streamed_text_frames( + text: &str, + chunks: &[&str], + usage: &Usage, + model: &ModelFixtureV1, +) -> Vec { + let message = assistant_message( + vec![ContentBlock::Text { + text: text.to_string(), + }], + StopReason::End, + Some(usage.clone()), + model, + 1, + ); + if chunks.is_empty() { + return vec![AssistantMessageEvent::Done { message }]; + } + + let mut frames = vec![ + AssistantMessageEvent::Start { + partial: assistant_message(Vec::new(), StopReason::End, None, model, 1), + }, + AssistantMessageEvent::TextStart { + partial: assistant_message( + vec![ContentBlock::Text { + text: String::new(), + }], + StopReason::End, + None, + model, + 1, + ), + }, + ]; + frames.extend(chunks.iter().map(|chunk| AssistantMessageEvent::TextDelta { + partial: None, + delta: (*chunk).to_string(), + })); + frames.extend([ + AssistantMessageEvent::TextEnd { + partial: assistant_message( + vec![ContentBlock::Text { + text: text.to_string(), + }], + StopReason::End, + None, + model, + 1, + ), + }, + AssistantMessageEvent::Usage { + usage: usage.clone(), + }, + AssistantMessageEvent::Stop { + stop_reason: StopReason::End, + error_message: None, + error_kind: None, + }, + AssistantMessageEvent::Done { message }, + ]); + frames +} + +pub(super) fn response( + stop_reason: StopReason, + usage: Usage, + model: &ModelFixtureV1, +) -> RouterChatResponse { + RouterChatResponse { + ok: true, + provider: model.provider.clone(), + model: model.id.clone(), + stop_reason: Some(stop_reason), + usage: Some(usage), + error: None, + } +} + +pub(super) fn usage(input: u64, output: u64) -> Usage { + Usage { + input: Some(input), + output: Some(output), + ..Default::default() + } +} + +pub(super) fn recorder(target: RecorderTargetV1) -> RecorderConfigV1 { + RecorderConfigV1 { + target, + lifecycle: RecorderLifecycleV1 { + trigger_type: LifecycleTriggerType::TurnCompleted, + function_id: LifecycleFunctionId::Lifecycle, + }, + } +} + +pub(super) fn synthetic_recorder() -> RecorderConfigV1 { + recorder(RecorderTargetV1 { + function_id: "{{run_id}}::unused".to_string(), + description: "Synthetic integration target; must never be called.".to_string(), + request_schema: json!({ + "type": "object", + "additionalProperties": false + }) + .as_object() + .expect("object") + .clone(), + response: json!({ + "content": [{ "type": "text", "text": "unused" }], + "is_error": false + }), + }) +} + +pub(super) fn recorder_target(function_id: &str) -> RecorderTargetV1 { + RecorderTargetV1 { + function_id: function_id.to_string(), + description: "Record one integration fixture value.".to_string(), + request_schema: json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + }) + .as_object() + .expect("object") + .clone(), + response: json!({ + "content": [{ "type": "text", "text": "recorded" }], + "is_error": false + }), + } +} + +pub(super) fn system_prompt(allowed_functions: &[String]) -> String { + let base = DEFAULT_SYSTEM_PROMPT + .strip_suffix('\n') + .unwrap_or(DEFAULT_SYSTEM_PROMPT); + let policy = if allowed_functions.is_empty() { + "Function dispatch is entirely disabled this turn — do not call any function.".to_string() + } else { + format!( + "Your dispatch policy allows ONLY these functions: {}. This narrowed-policy \ + instruction OVERRIDES the general discovery requirement for this turn: call the \ + listed target ids directly when the task already supplies their arguments. Anything \ + else — including discovery (engine::functions::list / ::info) unless listed above — \ + is denied. Do not probe: if the task genuinely needs an unlisted function or an \ + unknown contract, report that blocker and finish.", + allowed_functions.join(", ") + ) + }; + format!("{base}\n\nYour session id is {{{{session_id}}}}.\n{policy}") +} diff --git a/harness/evals/integration/src/stack/bins.rs b/harness/evals/integration/src/stack/bins.rs index 47a69e886..c55ea81e1 100644 --- a/harness/evals/integration/src/stack/bins.rs +++ b/harness/evals/integration/src/stack/bins.rs @@ -7,6 +7,7 @@ use super::config::WORKER_START_ORDER; pub struct StackBins { pub engine: PathBuf, pub harness: PathBuf, + pub console: Option, /// queue, iii-directory, session-manager, context-manager. pub workers: BTreeMap, } @@ -17,6 +18,7 @@ impl StackBins { match name { "engine" => Some(&self.engine), "harness" => Some(&self.harness), + "console" => self.console.as_deref(), other => self.workers.get(other).map(PathBuf::as_path), } } diff --git a/harness/evals/integration/src/stack/manifest.rs b/harness/evals/integration/src/stack/manifest.rs index e484ab95d..c9d8ce7a4 100644 --- a/harness/evals/integration/src/stack/manifest.rs +++ b/harness/evals/integration/src/stack/manifest.rs @@ -42,6 +42,9 @@ pub(crate) fn stack_info(bins: &StackBins, layout: &RunLayout, port: u16) -> any }; record("engine", &bins.engine)?; record("harness", &bins.harness)?; + if let Some(console) = &bins.console { + record("console", console)?; + } for (name, path) in &bins.workers { record(name, path)?; } diff --git a/harness/evals/integration/src/stack/supervisor.rs b/harness/evals/integration/src/stack/supervisor.rs index 62ca25604..d71065c4c 100644 --- a/harness/evals/integration/src/stack/supervisor.rs +++ b/harness/evals/integration/src/stack/supervisor.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::Path; use std::time::Duration; use crate::process::{ProcessSpec, ProcessSupervisor, TeardownReport, DEFAULT_TEARDOWN_BUDGET}; @@ -11,10 +11,6 @@ pub struct Stack { pub ws_url: String, pub paths: RunLayout, processes: ProcessSupervisor, - /// Engine spawn recipe, kept for fault-injection respawns: - /// (binary, args, cwd). `None` only in test stacks. - engine_recipe: Option<(PathBuf, Vec, PathBuf)>, - engine_restarts: u32, } #[derive(Debug)] @@ -69,12 +65,6 @@ impl Stack { ws_url, paths: paths.clone(), processes: ProcessSupervisor::default(), - engine_recipe: Some(( - bins.engine.clone(), - engine_args.clone(), - paths.engine_dir.clone(), - )), - engine_restarts: 0, }; if let Err(error) = @@ -120,27 +110,18 @@ impl Stack { self.spawn_worker("harness", &bins.harness) } - pub async fn kill_engine(&mut self) -> anyhow::Result<()> { - let mut engine = self - .processes - .remove("engine") - .ok_or_else(|| anyhow::anyhow!("no live engine child to kill"))?; - engine.kill_now().await?; - tracing::info!( - target: "harness_integration::stack", - "engine SIGKILLed (fault injection)" - ); - Ok(()) - } - - pub fn respawn_engine(&mut self) -> anyhow::Result<()> { - let (bin, args, cwd) = self - .engine_recipe - .clone() - .ok_or_else(|| anyhow::anyhow!("stack has no engine recipe (test stack?)"))?; - self.engine_restarts += 1; - let log_name = format!("engine.restart{}", self.engine_restarts); - self.spawn_child_logged("engine", &log_name, &bin, &args, &cwd) + /// Spawn the production Console on a run-scoped loopback port. + pub fn spawn_console(&mut self, bin: &Path) -> anyhow::Result { + let port = free_loopback_port()?; + let args = vec![ + "--url".to_string(), + self.ws_url.clone(), + "--http-port".to_string(), + port.to_string(), + ]; + let root = self.paths.root.clone(); + self.spawn_child("console", bin, &args, &root)?; + Ok(format!("http://127.0.0.1:{port}")) } fn spawn_worker(&mut self, worker: &str, bin: &Path) -> anyhow::Result<()> { @@ -159,8 +140,6 @@ impl Stack { ws_url: "ws://127.0.0.1:0".to_string(), paths, processes: ProcessSupervisor::new(DEFAULT_TEARDOWN_BUDGET), - engine_recipe: None, - engine_restarts: 0, } } diff --git a/harness/evals/integration/src/stack/tests.rs b/harness/evals/integration/src/stack/tests.rs index 56ede651a..643c8d08c 100644 --- a/harness/evals/integration/src/stack/tests.rs +++ b/harness/evals/integration/src/stack/tests.rs @@ -51,6 +51,7 @@ fn manifest_is_canonical_and_uses_layout_paths() { let bins = StackBins { engine: binary.clone(), harness: binary.clone(), + console: None, workers: BTreeMap::from([("queue".to_string(), PathBuf::from(&binary))]), }; @@ -75,6 +76,7 @@ fn manifest_fails_when_a_binary_cannot_be_identified() { let bins = StackBins { engine: missing.clone(), harness: missing.clone(), + console: None, workers: BTreeMap::new(), }; let error = manifest::stack_info(&bins, &layout, 3210).unwrap_err(); @@ -88,6 +90,7 @@ async fn boot_failure_carries_a_complete_typed_teardown() { let bins = StackBins { engine: PathBuf::from("/bin/true"), harness: PathBuf::from("/bin/true"), + console: None, workers: BTreeMap::new(), }; diff --git a/harness/evals/integration/src/types/recorder.rs b/harness/evals/integration/src/types/recorder.rs index 921829d8b..4b331cb71 100644 --- a/harness/evals/integration/src/types/recorder.rs +++ b/harness/evals/integration/src/types/recorder.rs @@ -12,12 +12,6 @@ use super::script::SchemaVersion1; pub struct RecorderConfigV1 { pub target: RecorderTargetV1, pub lifecycle: RecorderLifecycleV1, - /// Additional run-scoped controlled functions (e.g. hook implementations - /// for `harness::hook::*` scenarios). Registered exactly like the target: - /// declared description/schema verbatim, every call durably recorded, - /// declared response returned. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub extra_functions: Vec, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -31,12 +25,6 @@ pub struct RecorderTargetV1 { pub request_schema: serde_json::Map, /// Declared response returned for every target call. pub response: serde_json::Value, - /// One-based call ordinal whose response is held behind a runner-owned, - /// in-process gate after the call has been durably appended. Derived from - /// `fault.after_target_calls`; never authored directly. - #[serde(skip_serializing_if = "Option::is_none")] - #[schemars(range(min = 1))] - pub hold_response_at: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] diff --git a/harness/evals/integration/src/types/scenario.rs b/harness/evals/integration/src/types/scenario.rs index 4319baa66..32060527e 100644 --- a/harness/evals/integration/src/types/scenario.rs +++ b/harness/evals/integration/src/types/scenario.rs @@ -1,18 +1,8 @@ -//! Authored integration-scenario V1, its strict compiled runtime form, and -//! the stable result contract. -//! -//! The checked-in contract intentionally contains only scenario intent. -//! [`crate::expand::compile_scenario`] derives run-scoped ids, the exact -//! `harness::send` payload, recorder configuration, and router -//! matchers/frames before the stack starts. Run outcomes are checked by the -//! runner-owned floor and each scenario's `verify` function over -//! [`crate::evidence_data::RunEvidence`]. +//! Strict scenario runtime and result contracts. -mod authored; mod compiled; mod result; -pub use authored::*; pub use compiled::*; pub use result::*; @@ -21,7 +11,7 @@ mod tests { use super::*; #[test] - fn authored_defaults_are_safe() { + fn deadline_defaults_are_safe() { let timeouts = DeadlinesV1::default(); assert_eq!(timeouts.readiness_ms, 60_000); assert_eq!(timeouts.scenario_ms, 60_000); diff --git a/harness/evals/integration/src/types/scenario/authored.rs b/harness/evals/integration/src/types/scenario/authored.rs deleted file mode 100644 index 374df9e97..000000000 --- a/harness/evals/integration/src/types/scenario/authored.rs +++ /dev/null @@ -1,184 +0,0 @@ -use std::collections::BTreeMap; - -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::types::frames::Usage; -use crate::types::script::{JsonMatcherV1, ModelFixtureV1}; - -/// The authored scenario data built by the `src/scenarios` builder modules. -/// -/// This layer is code, never serialized: there is no schema pair to keep -/// synchronized and no round trip. `DeadlinesV1`, `ReleaseV1`, and -/// `FaultKind` below are shared with the compiled layer and keep their wire -/// derives. -#[derive(Debug, Clone, PartialEq)] -pub struct AuthoredScenarioV1 { - pub id: String, - pub description: String, - pub quarantine: bool, - pub send: ScenarioSendV1, - /// Alias → controlled function. Aliases are expanded to - /// `{{run_id}}::` by the compiler. - pub functions: BTreeMap, - pub router: ScenarioRouterV1, - pub bindings: Vec, - pub release: Option, - pub fault: Option, - pub timeouts: DeadlinesV1, -} - -/// Scenario ids are also artifact directory names, so keep them to one safe, -/// portable path component. -pub fn validate_scenario_id(id: &str) -> anyhow::Result<()> { - let valid = !id.is_empty() - && id.len() <= 128 - && id - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')); - anyhow::ensure!( - valid, - "scenario id must be 1-128 ASCII letters, digits, '-' or '_'" - ); - Ok(()) -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ScenarioSendV1 { - pub message: String, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ScenarioFunctionV1 { - pub description: String, - pub request_schema: serde_json::Map, - pub response: serde_json::Value, - /// Exposed to the model by default. Hook-only controlled functions set - /// this to false. - pub expose: bool, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ScenarioRouterV1 { - /// Omitted for the deterministic `fixture-model` / `scripted` catalog - /// entry used by the integration stack. - pub model: Option, - pub generations: Vec, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ScenarioGenerationV1 { - pub reply: RouterReplyV1, - /// Escape hatch for fields whose history is intentionally unstable, such - /// as the post-crash request in a recovery reproduction. - pub match_overrides: GenerationMatchOverridesV1, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum RouterReplyV1 { - Text { - text: String, - /// Non-empty chunks produce the complete streaming frame sequence. - /// Omitted chunks produce one terminal `done` frame. - chunks: Vec, - usage: Option, - }, - FunctionCall { - /// Defaults to `call-`. - id: Option, - /// Function alias from `functions`. - function: String, - arguments: serde_json::Value, - usage: Option, - }, -} - -#[derive(Debug, Clone, Default, PartialEq)] -pub struct GenerationMatchOverridesV1 { - pub request_id: Option, - pub system_prompt: Option, - pub messages: Option, - pub tools: Option, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct TriggerBindingSpecV1 { - pub trigger: TriggerKindV1, - /// Controlled function alias invoked by the trigger. - pub function: String, - /// Exposed function aliases selected by this hook. - pub functions: Vec, - pub priority: i64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TriggerKindV1 { - HookPreTrigger, -} - -impl TriggerKindV1 { - pub fn as_str(self) -> &'static str { - match self { - Self::HookPreTrigger => "harness::hook::pre-trigger", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FaultV1 { - pub kind: FaultKind, - pub after_target_calls: u64, - pub restart_delay_ms: u64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub enum FaultKind { - EngineSigkill, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct ReleaseV1 { - #[serde( - default = "default_call_id", - skip_serializing_if = "is_default_call_id" - )] - pub function_call_id: String, - pub action: ReleaseActionV1, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub enum ReleaseActionV1 { - Execute, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct DeadlinesV1 { - #[schemars(range(min = 1))] - pub readiness_ms: u64, - #[schemars(range(min = 1))] - pub scenario_ms: u64, - #[schemars(range(min = 1))] - pub teardown_ms: u64, -} - -impl Default for DeadlinesV1 { - fn default() -> Self { - Self { - readiness_ms: 60_000, - scenario_ms: 60_000, - teardown_ms: 15_000, - } - } -} - -pub(super) fn default_call_id() -> String { - "call-1".to_string() -} - -fn is_default_call_id(value: &str) -> bool { - value == default_call_id() -} diff --git a/harness/evals/integration/src/types/scenario/compiled.rs b/harness/evals/integration/src/types/scenario/compiled.rs index e71952242..d7107bebe 100644 --- a/harness/evals/integration/src/types/scenario/compiled.rs +++ b/harness/evals/integration/src/types/scenario/compiled.rs @@ -4,9 +4,20 @@ use serde::{Deserialize, Serialize}; use crate::types::recorder::RecorderConfigV1; use crate::types::script::SchemaVersion1; -use super::{DeadlinesV1, FaultKind, ReleaseV1}; +pub fn validate_scenario_id(id: &str) -> anyhow::Result<()> { + let valid = !id.is_empty() + && id.len() <= 128 + && id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')); + anyhow::ensure!( + valid, + "scenario id must be 1-128 ASCII letters, digits, '-' or '_'" + ); + Ok(()) +} -/// Strict runtime scenario produced from [`super::AuthoredScenarioV1`]. +/// Strict runtime scenario consumed by the integration runner. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CompiledScenarioV1 { @@ -17,12 +28,6 @@ pub struct CompiledScenarioV1 { pub send: CompiledSendV1, pub recorder: RecorderConfigV1, pub deadlines: DeadlinesV1, - #[serde(skip_serializing_if = "Option::is_none")] - pub fault: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub bindings: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub release: Option, } /// The deliberately narrow `harness::send` request emitted by the scenario @@ -59,20 +64,23 @@ pub enum CompiledFunctionExposureV1 { Native, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] -pub struct CompiledFaultV1 { - pub kind: FaultKind, - pub function_id: String, +pub struct DeadlinesV1 { + #[schemars(range(min = 1))] + pub readiness_ms: u64, #[schemars(range(min = 1))] - pub after_target_calls: u64, - pub restart_delay_ms: u64, + pub scenario_ms: u64, + #[schemars(range(min = 1))] + pub teardown_ms: u64, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct TriggerBindingV1 { - pub trigger_type: String, - pub function_id: String, - pub config: serde_json::Value, +impl Default for DeadlinesV1 { + fn default() -> Self { + Self { + readiness_ms: 60_000, + scenario_ms: 60_000, + teardown_ms: 15_000, + } + } } diff --git a/harness/evals/integration/src/types/script.rs b/harness/evals/integration/src/types/script.rs index 58c7b2c00..ab2862361 100644 --- a/harness/evals/integration/src/types/script.rs +++ b/harness/evals/integration/src/types/script.rs @@ -48,7 +48,6 @@ pub struct ModelFixtureV1 { #[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] pub enum JsonMatcherV1 { Absent, - Present, Regex { pattern: String, }, diff --git a/harness/evals/integration/tests/scenario_compilation.rs b/harness/evals/integration/tests/scenario_compilation.rs index 303c6f1af..6db0a1596 100644 --- a/harness/evals/integration/tests/scenario_compilation.rs +++ b/harness/evals/integration/tests/scenario_compilation.rs @@ -1,20 +1,12 @@ -use harness_integration::fixtures::ScenarioFixture; -use harness_integration::scenarios::RegisteredScenario; use harness_integration::types::script::JsonMatcherV1; -fn load(entry: &RegisteredScenario) -> ScenarioFixture { - ScenarioFixture::from_registered(entry) - .unwrap_or_else(|error| panic!("compiling {}: {error:#}", entry.slug)) -} - #[test] fn inferred_function_history_contains_call_and_result() { - let registered = harness_integration::scenarios::all(); - let entry = registered + let fixtures = harness_integration::scenarios::all(); + let fixture = fixtures .iter() - .find(|entry| entry.slug == "exactly-once-function") + .find(|fixture| fixture.slug == "exactly-once-function") .expect("exactly-once-function is registered"); - let fixture = load(entry); let matcher = &fixture.script.generations[1].match_.messages; let JsonMatcherV1::Exact { expected, .. } = matcher else { panic!("function history should use an exact matcher"); diff --git a/harness/evals/integration/tests/schemas.rs b/harness/evals/integration/tests/schemas.rs index 5b2a02f0d..ab1dd123d 100644 --- a/harness/evals/integration/tests/schemas.rs +++ b/harness/evals/integration/tests/schemas.rs @@ -2,7 +2,7 @@ //! router, recorder-event, and result), plus validation of compiled payloads //! against the producer-owned contracts. There are no generated goldens: //! wire shape is pinned by the round trips, and silent compiler weakening is -//! caught by the property tests in `scenario_compilation.rs`. +//! caught by focused fixture tests. use harness_integration::expand::CompiledFixtureV1; use harness_integration::types::recorder::{RecorderEventKind, RecorderEventV1}; @@ -11,17 +11,13 @@ use harness_integration::types::scenario::{ }; use harness_integration::types::script::{RouterScriptV1, SchemaVersion1}; -/// Every registered scenario compiles and its strict runtime representation -/// round-trips through its typed mirror. The authored layer is code and has -/// no round trip. +/// Every strict checked-in fixture round-trips through its typed mirror. #[test] fn registered_scenarios_compile_and_round_trip() { - use harness_integration::fixtures::ScenarioFixture; - - let registered = harness_integration::scenarios::all(); - assert!(!registered.is_empty(), "expected at least one scenario"); - for entry in ®istered { - let fixture = ScenarioFixture::from_registered(entry).unwrap(); + let fixtures = harness_integration::scenarios::all(); + assert!(!fixtures.is_empty(), "expected at least one scenario"); + for fixture in &fixtures { + fixture.validate().unwrap(); let compiled_value = serde_json::to_value(&fixture.scenario).unwrap(); let compiled_again: CompiledScenarioV1 = serde_json::from_value(compiled_value).unwrap(); @@ -85,16 +81,13 @@ fn evidence_and_report_contracts_round_trip() { #[test] fn compiled_send_is_accepted_by_the_authoritative_harness_contract() { - use harness_integration::fixtures::ScenarioFixture; - let golden: serde_json::Value = serde_json::from_str(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/../../../harness/tests/golden/schemas/harness.send.json" ))) .unwrap(); let validator = jsonschema::JSONSchema::compile(&golden["request_schema"]).unwrap(); - for entry in &harness_integration::scenarios::all() { - let fixture = ScenarioFixture::from_registered(entry).unwrap(); + for fixture in &harness_integration::scenarios::all() { let send = serde_json::to_value(&fixture.scenario.send).unwrap(); let errors = validator .validate(&send) @@ -111,24 +104,23 @@ fn compiled_send_is_accepted_by_the_authoritative_harness_contract() { #[test] fn compiled_schema_matches_runtime_safety_constraints() { - use harness_integration::expand::{compile_scenario, scenario_template, ScenarioTemplateKind}; - let schema = serde_json::to_value(schemars::schema_for!(CompiledScenarioV1)).unwrap(); let validator = jsonschema::JSONSchema::compile(&schema).unwrap(); - let authored = scenario_template( - "E2E-COMPILED-SCHEMA", - "Validate compiled schema constraints.", - ScenarioTemplateKind::Crash, - ); - let valid = - serde_json::to_value(compile_scenario(&authored, "prompt").unwrap().scenario).unwrap(); + let valid = serde_json::to_value( + harness_integration::scenarios::all() + .into_iter() + .next() + .unwrap() + .scenario, + ) + .unwrap(); assert!(validator.is_valid(&valid)); let mut unsafe_id = valid.clone(); unsafe_id["id"] = serde_json::json!("../../escape"); assert!(!validator.is_valid(&unsafe_id)); - let mut zero_fault_threshold = valid; - zero_fault_threshold["fault"]["after_target_calls"] = serde_json::json!(0); - assert!(!validator.is_valid(&zero_fault_threshold)); + let mut zero_deadline = valid; + zero_deadline["deadlines"]["scenario_ms"] = serde_json::json!(0); + assert!(!validator.is_valid(&zero_deadline)); } From 73d3129b39edc44234a1163086481a1991fa2b74 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Wed, 22 Jul 2026 08:19:53 -0300 Subject: [PATCH 3/5] (MOT-4107) refactor(integration): add typed scenario DSL --- harness/evals/integration/README.md | 11 +- .../evals/integration/src/evidence_data.rs | 83 ++ .../src/scenarios/console_streamed_text.rs | 79 +- .../evals/integration/src/scenarios/dsl.rs | 743 ++++++++++++++++++ .../src/scenarios/exactly_once_function.rs | 191 ++--- .../evals/integration/src/scenarios/mod.rs | 2 +- .../src/scenarios/streamed_text.rs | 93 +-- .../integration/src/scenarios/support.rs | 314 -------- 8 files changed, 969 insertions(+), 547 deletions(-) create mode 100644 harness/evals/integration/src/scenarios/dsl.rs delete mode 100644 harness/evals/integration/src/scenarios/support.rs diff --git a/harness/evals/integration/README.md b/harness/evals/integration/README.md index 70c23d946..7d613624b 100644 --- a/harness/evals/integration/README.md +++ b/harness/evals/integration/README.md @@ -15,11 +15,12 @@ No provider key or network access is required. | E2E-002 | `exactly-once-function` | direct | a native function executes exactly once | | UI-001 | `console-streamed-text` | playground | a message sent by the Console streams to durable completion | -Each fixture is defined end to end in its own `src/scenarios/*.rs` file: the -exact `harness::send` payload, router request matchers, response frames, -recorder configuration, and scenario-specific verification function. Shared -code is limited to wire-format constructors. There is no YAML or generic -authored-scenario compiler. +Each fixture is defined end to end in its own `src/scenarios/*.rs` file with a +small typed DSL. The scenario keeps its send policy, router request matchers, +response behavior, recorder configuration, function history, and verification +visible at the call site. Builders compile directly to the runtime types; there +is no YAML, macro layer, inferred history, or generic authored-scenario +compiler. ## Run the direct scenarios diff --git a/harness/evals/integration/src/evidence_data.rs b/harness/evals/integration/src/evidence_data.rs index 495c99454..8b54c1577 100644 --- a/harness/evals/integration/src/evidence_data.rs +++ b/harness/evals/integration/src/evidence_data.rs @@ -89,6 +89,67 @@ impl RunEvidence { .any(|entry_id| !seen.insert(entry_id)) } + pub fn expect_assistant_texts( + &self, + expected: impl IntoIterator>, + ) -> anyhow::Result<()> { + let expected: Vec = expected + .into_iter() + .map(|text| text.as_ref().to_string()) + .collect(); + let actual = self.assistant_texts(); + anyhow::ensure!( + actual == expected, + "assistant texts {actual:?} != {expected:?}" + ); + Ok(()) + } + + pub fn expect_message_counts( + &self, + user: u64, + assistant: u64, + function_result: u64, + ) -> anyhow::Result<()> { + let actual = self.message_counts(); + let expected = (user, assistant, function_result); + anyhow::ensure!( + actual == expected, + "message counts (user, assistant, function_result) {actual:?} != {expected:?}" + ); + Ok(()) + } + + pub fn expect_function_calls(&self, alias: &str, count: usize) -> anyhow::Result<()> { + let actual = self.calls(alias).len(); + anyhow::ensure!( + actual == count, + "{alias} ran {actual} times, expected {count}" + ); + Ok(()) + } + + pub fn expect_call_payload(&self, alias: &str, expected: Value) -> anyhow::Result<()> { + let calls = self.calls(alias); + let call = calls + .first() + .ok_or_else(|| anyhow::anyhow!("{alias} did not run"))?; + anyhow::ensure!( + call.payload == expected, + "{alias} payload {} != {expected}", + call.payload + ); + Ok(()) + } + + pub fn expect_no_duplicate_messages(&self) -> anyhow::Result<()> { + anyhow::ensure!( + !self.has_duplicate_messages(), + "transcript contains duplicate entry ids" + ); + Ok(()) + } + /// Replace this run's concrete ids with `{{run_id}}` / `{{session_id}}` / /// `{{turn_id}}` placeholders so persisted failure text stays /// byte-comparable across runs. @@ -238,4 +299,26 @@ mod tests { evidence.turn_id = None; assert_eq!(evidence.scrub("turn t_1"), "turn t_1"); } + + #[test] + fn expectation_helpers_report_the_observed_values() { + let mut evidence = base_evidence(); + evidence.transcript = vec![ + json!({ "message": { "role": "user", "content": [] } }), + json!({ + "message": { + "role": "assistant", + "content": [{ "type": "text", "text": "complete" }] + } + }), + ]; + evidence.expect_assistant_texts(["complete"]).unwrap(); + evidence.expect_message_counts(1, 1, 0).unwrap(); + evidence.expect_no_duplicate_messages().unwrap(); + assert!(evidence + .expect_assistant_texts(["different"]) + .unwrap_err() + .to_string() + .contains("complete")); + } } diff --git a/harness/evals/integration/src/scenarios/console_streamed_text.rs b/harness/evals/integration/src/scenarios/console_streamed_text.rs index 341f2a460..8754df9dd 100644 --- a/harness/evals/integration/src/scenarios/console_streamed_text.rs +++ b/harness/evals/integration/src/scenarios/console_streamed_text.rs @@ -1,60 +1,45 @@ //! UI-001 — Playwright sends a Console turn and validates the rendered result. -use anyhow::ensure; -use serde_json::json; - -use super::support::{ - model, request_match, response, send, streamed_text_frames, synthetic_recorder, system_prompt, - usage, user_message, RequestProfile, -}; +use super::dsl::{Generation, Message, Model, Recorder, Request, Response, Scenario, Send, Tool}; use super::ScenarioDriver; use crate::fixtures::ScenarioFixture; -use crate::types::frames::StopReason; -use crate::types::scenario::{CompiledScenarioV1, DeadlinesV1}; -use crate::types::script::{RouterScriptV1, SchemaVersion1, ScriptedGenerationV1}; pub(super) fn scenario() -> ScenarioFixture { const ID: &str = "UI-001"; const MESSAGE: &str = "Return the console fixture phrase."; const TEXT: &str = "console fixture complete"; - let model = model(); - let usage = usage(9, 3); - let allowed_functions = Vec::new(); - let messages = vec![user_message(MESSAGE)]; - let generation = ScriptedGenerationV1 { - ordinal: 1, - match_: request_match(1, &model, &messages, &json!([]), RequestProfile::Console), - frames: streamed_text_frames(TEXT, &["console fixture ", "complete"], &usage, &model), - response: response(StopReason::End, usage, &model), - }; - - ScenarioFixture { - slug: "console-streamed-text".to_string(), - driver: ScenarioDriver::Playground, - scenario: CompiledScenarioV1 { - schema_version: SchemaVersion1::V1, - id: ID.to_string(), - description: "A Console-sent streamed turn reaches durable completion.".to_string(), - send: send(ID, MESSAGE, &model, &allowed_functions), - recorder: synthetic_recorder(), - deadlines: DeadlinesV1::default(), - }, - script: RouterScriptV1 { - schema_version: SchemaVersion1::V1, - scenario_id: ID.to_string(), - model, - generations: vec![generation], - }, - system_prompt_template: system_prompt(&allowed_functions), - verify: |run| { - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" - ); - Ok(()) - }, - } + Scenario::new( + ID, + "console-streamed-text", + "A Console-sent streamed turn reaches durable completion.", + ScenarioDriver::Playground, + Model::scripted("fixture-model"), + ) + .send( + Send::message(MESSAGE) + .idempotency_key("{{run_id}}:ui-001") + .without_functions(), + ) + .recorder(Recorder::lifecycle_only()) + .generation( + Generation::new(1) + .expect( + Request::new() + .turn_request() + .system_prompt_regex("agent_trigger") + .messages_exact([Message::user(MESSAGE)]) + .tools_subset([Tool::named("agent_trigger")]), + ) + .respond(Response::streamed_text( + TEXT, + ["console fixture ", "complete"], + 9, + 3, + )), + ) + .verify(|run| run.expect_no_duplicate_messages()) + .build() } #[cfg(test)] diff --git a/harness/evals/integration/src/scenarios/dsl.rs b/harness/evals/integration/src/scenarios/dsl.rs new file mode 100644 index 000000000..4255424bb --- /dev/null +++ b/harness/evals/integration/src/scenarios/dsl.rs @@ -0,0 +1,743 @@ +//! Small typed vocabulary for authoring checked-in scenarios. +//! +//! Builders remove wire-format repetition but compile directly to the runtime +//! types. Request differences and function history remain explicit at each +//! scenario call site. + +use serde_json::{json, Value}; + +use super::{ScenarioDriver, VerifyFn}; +use crate::fixtures::ScenarioFixture; +use crate::types::frames::{ + AssistantMessage, AssistantMessageEvent, AssistantRoleTag, ContentBlock, RouterChatResponse, + StopReason, Usage, +}; +use crate::types::recorder::{ + LifecycleFunctionId, LifecycleTriggerType, RecorderConfigV1, RecorderLifecycleV1, + RecorderTargetV1, +}; +use crate::types::scenario::{ + CompiledFunctionExposureV1, CompiledFunctionPolicyV1, CompiledScenarioV1, + CompiledSendOptionsV1, CompiledSendV1, DeadlinesV1, +}; +use crate::types::script::{ + GenerationMatchV1, JsonMatcherV1, JsonNormalizerV1, ModelFixtureV1, NormalizerOperation, + RouterScriptV1, SchemaVersion1, ScriptedGenerationV1, +}; + +const DEFAULT_SYSTEM_PROMPT: &str = include_str!("../../../../prompts/default.txt"); +const PROVIDER: &str = "scripted"; + +pub(super) struct Model; + +impl Model { + pub(super) fn scripted(id: &str) -> ModelFixtureV1 { + ModelFixtureV1 { + id: id.to_string(), + provider: PROVIDER.to_string(), + context_window: 32_768, + max_output_tokens: 4_096, + supports_thinking: Some(false), + supports_xhigh: None, + supports_tools: Some(true), + supports_vision: Some(false), + supports_cache: Some(false), + supports_structured_output: Some(true), + } + } +} + +pub(super) struct Scenario { + id: String, + slug: String, + description: String, + driver: ScenarioDriver, + model: ModelFixtureV1, + send: Option, + recorder: Option, + generations: Vec, + verify: Option, +} + +impl Scenario { + pub(super) fn new( + id: &str, + slug: &str, + description: &str, + driver: ScenarioDriver, + model: ModelFixtureV1, + ) -> Self { + Self { + id: id.to_string(), + slug: slug.to_string(), + description: description.to_string(), + driver, + model, + send: None, + recorder: None, + generations: Vec::new(), + verify: None, + } + } + + pub(super) fn send(mut self, send: Send) -> Self { + self.send = Some(send); + self + } + + pub(super) fn recorder(mut self, recorder: RecorderConfigV1) -> Self { + self.recorder = Some(recorder); + self + } + + pub(super) fn generation(mut self, generation: Generation) -> Self { + self.generations.push(generation); + self + } + + pub(super) fn verify(mut self, verify: VerifyFn) -> Self { + self.verify = Some(verify); + self + } + + pub(super) fn build(self) -> ScenarioFixture { + let send = self.send.expect("scenario send is required"); + let allowed_functions = send.allowed_functions.clone(); + let compiled_send = send.compile(&self.id, &self.model); + let recorder = self.recorder.expect("scenario recorder is required"); + let generations = self + .generations + .into_iter() + .map(|generation| generation.compile(&self.model)) + .collect(); + + ScenarioFixture { + slug: self.slug, + driver: self.driver, + scenario: CompiledScenarioV1 { + schema_version: SchemaVersion1::V1, + id: self.id.clone(), + description: self.description, + send: compiled_send, + recorder, + deadlines: DeadlinesV1::default(), + }, + script: RouterScriptV1 { + schema_version: SchemaVersion1::V1, + scenario_id: self.id, + model: self.model, + generations, + }, + system_prompt_template: system_prompt(&allowed_functions), + verify: self.verify.expect("scenario verification is required"), + } + } +} + +pub(super) struct Send { + message: String, + idempotency_key: Option, + allowed_functions: Vec, +} + +impl Send { + pub(super) fn message(message: &str) -> Self { + Self { + message: message.to_string(), + idempotency_key: None, + allowed_functions: Vec::new(), + } + } + + pub(super) fn idempotency_key(mut self, key: &str) -> Self { + self.idempotency_key = Some(key.to_string()); + self + } + + pub(super) fn without_functions(self) -> Self { + self + } + + pub(super) fn allow_function(mut self, function: &RecorderFunction) -> Self { + if !self + .allowed_functions + .contains(&function.target.function_id) + { + self.allowed_functions + .push(function.target.function_id.clone()); + } + self + } + + fn compile(self, scenario_id: &str, model: &ModelFixtureV1) -> CompiledSendV1 { + CompiledSendV1 { + session_id: "{{session_id}}".to_string(), + message: self.message, + model: model.id.clone(), + provider: model.provider.clone(), + idempotency_key: self + .idempotency_key + .unwrap_or_else(|| format!("{{{{run_id}}}}:{}", scenario_id.to_ascii_lowercase())), + options: CompiledSendOptionsV1 { + functions: CompiledFunctionPolicyV1 { + allow: self.allowed_functions, + deny: Vec::new(), + expose: CompiledFunctionExposureV1::Native, + }, + }, + } + } +} + +pub(super) struct Recorder; + +impl Recorder { + pub(super) fn lifecycle_only() -> RecorderConfigV1 { + recorder_config(RecorderTargetV1 { + function_id: "{{run_id}}::unused".to_string(), + description: "Synthetic integration target; must never be called.".to_string(), + request_schema: json!({ + "type": "object", + "additionalProperties": false + }) + .as_object() + .expect("object") + .clone(), + response: json!({ + "content": [{ "type": "text", "text": "unused" }], + "is_error": false + }), + }) + } + + pub(super) fn function(function: RecorderFunction) -> RecorderConfigV1 { + recorder_config(function.target) + } +} + +#[derive(Clone)] +pub(super) struct RecorderFunction { + target: RecorderTargetV1, +} + +impl RecorderFunction { + pub(super) fn new(function_id: &str, description: &str) -> Self { + Self { + target: RecorderTargetV1 { + function_id: function_id.to_string(), + description: description.to_string(), + request_schema: serde_json::Map::new(), + response: Value::Null, + }, + } + } + + pub(super) fn request_schema(mut self, schema: Value) -> Self { + self.target.request_schema = schema + .as_object() + .unwrap_or_else(|| panic!("recorder request schema must be an object: {schema}")) + .clone(); + self + } + + pub(super) fn returns_text(mut self, text: &str) -> Self { + self.target.response = json!({ + "content": [{ "type": "text", "text": text }], + "is_error": false + }); + self + } + + pub(super) fn id(&self) -> &str { + &self.target.function_id + } + + pub(super) fn tool(&self) -> Value { + json!({ + "name": self.target.function_id, + "description": self.target.description, + "parameters": self.target.request_schema, + "execution_mode": "sequential" + }) + } +} + +pub(super) struct Request { + turn_request: bool, + system_prompt: Option, + messages: Option, + tools: Option, +} + +impl Request { + pub(super) fn new() -> Self { + Self { + turn_request: false, + system_prompt: None, + messages: None, + tools: None, + } + } + + pub(super) fn turn_request(mut self) -> Self { + self.turn_request = true; + self + } + + pub(super) fn system_prompt_sha256(mut self, expected: &str) -> Self { + self.system_prompt = Some(JsonMatcherV1::Sha256 { + expected: expected.to_string(), + }); + self + } + + pub(super) fn system_prompt_regex(mut self, pattern: &str) -> Self { + self.system_prompt = Some(JsonMatcherV1::Regex { + pattern: pattern.to_string(), + }); + self + } + + pub(super) fn messages_exact(mut self, messages: impl IntoIterator) -> Self { + let messages: Vec = messages.into_iter().collect(); + let normalize = (0..messages.len()) + .map(|index| JsonNormalizerV1 { + pointer: format!("/{index}/timestamp"), + operation: NormalizerOperation::Delete, + }) + .collect(); + self.messages = Some(JsonMatcherV1::Exact { + expected: Value::Array(messages), + normalize: Some(normalize), + }); + self + } + + pub(super) fn without_tools(mut self) -> Self { + self.tools = Some(exact(json!([]))); + self + } + + pub(super) fn tools_exact(mut self, tools: impl IntoIterator) -> Self { + self.tools = Some(exact(Value::Array(tools.into_iter().collect()))); + self + } + + pub(super) fn tools_subset(mut self, tools: impl IntoIterator) -> Self { + self.tools = Some(JsonMatcherV1::Subset { + expected: Value::Array(tools.into_iter().collect()), + normalize: None, + }); + self + } + + fn compile(self, model: &ModelFixtureV1, ordinal: u64) -> GenerationMatchV1 { + assert!(self.turn_request, "request id matcher is required"); + GenerationMatchV1 { + writer_ref: JsonMatcherV1::Subset { + expected: json!({ "direction": "write" }), + normalize: None, + }, + request_id: JsonMatcherV1::Regex { + pattern: if ordinal == 1 { + "^t_[0-9a-f]{32}:[0-9]+$".to_string() + } else { + format!("^t_[0-9a-f]{{32}}:{}$", ordinal - 1) + }, + }, + model: exact(json!(model.id)), + provider: exact(json!(model.provider)), + system_prompt: self + .system_prompt + .expect("system prompt matcher is required"), + messages: self.messages.expect("message matcher is required"), + tools: self.tools.expect("tool matcher is required"), + response_format: JsonMatcherV1::Absent, + thinking_level: JsonMatcherV1::Absent, + max_output_tokens: JsonMatcherV1::Absent, + provider_options: JsonMatcherV1::Absent, + metadata: JsonMatcherV1::Absent, + } + } +} + +pub(super) struct Generation { + ordinal: u64, + request: Option, + response: Option, +} + +impl Generation { + pub(super) fn new(ordinal: u64) -> Self { + Self { + ordinal, + request: None, + response: None, + } + } + + pub(super) fn expect(mut self, request: Request) -> Self { + self.request = Some(request); + self + } + + pub(super) fn respond(mut self, response: Response) -> Self { + self.response = Some(response); + self + } + + fn compile(self, model: &ModelFixtureV1) -> ScriptedGenerationV1 { + let (frames, response) = self + .response + .expect("generation response is required") + .compile(model, self.ordinal); + ScriptedGenerationV1 { + ordinal: self.ordinal, + match_: self + .request + .expect("generation request is required") + .compile(model, self.ordinal), + frames, + response, + } + } +} + +pub(super) struct Response { + kind: ResponseKind, + usage: Usage, +} + +enum ResponseKind { + StreamedText { + text: String, + chunks: Vec, + }, + Text(String), + FunctionCall { + call_id: String, + function_id: String, + arguments: Value, + }, +} + +impl Response { + pub(super) fn streamed_text( + text: &str, + chunks: I, + input_tokens: u64, + output_tokens: u64, + ) -> Self + where + I: IntoIterator, + S: Into, + { + Self { + kind: ResponseKind::StreamedText { + text: text.to_string(), + chunks: chunks.into_iter().map(Into::into).collect(), + }, + usage: usage(input_tokens, output_tokens), + } + } + + pub(super) fn text(text: &str, input_tokens: u64, output_tokens: u64) -> Self { + Self { + kind: ResponseKind::Text(text.to_string()), + usage: usage(input_tokens, output_tokens), + } + } + + pub(super) fn function_call( + call_id: &str, + function: &RecorderFunction, + arguments: Value, + input_tokens: u64, + output_tokens: u64, + ) -> Self { + Self { + kind: ResponseKind::FunctionCall { + call_id: call_id.to_string(), + function_id: function.id().to_string(), + arguments, + }, + usage: usage(input_tokens, output_tokens), + } + } + + fn compile( + self, + model: &ModelFixtureV1, + ordinal: u64, + ) -> (Vec, RouterChatResponse) { + let usage = self.usage; + let timestamp = i64::try_from(ordinal).expect("generation ordinal fits i64"); + let (frames, stop_reason) = match self.kind { + ResponseKind::StreamedText { text, chunks } => ( + streamed_text_frames(&text, &chunks, &usage, model, timestamp), + StopReason::End, + ), + ResponseKind::Text(text) => ( + vec![AssistantMessageEvent::Done { + message: assistant_message( + vec![ContentBlock::Text { text }], + StopReason::End, + Some(usage.clone()), + model, + timestamp, + ), + }], + StopReason::End, + ), + ResponseKind::FunctionCall { + call_id, + function_id, + arguments, + } => ( + vec![AssistantMessageEvent::Done { + message: assistant_message( + vec![ContentBlock::FunctionCall { + id: call_id, + function_id, + arguments, + }], + StopReason::FunctionCall, + Some(usage.clone()), + model, + timestamp, + ), + }], + StopReason::FunctionCall, + ), + }; + let response = RouterChatResponse { + ok: true, + provider: model.provider.clone(), + model: model.id.clone(), + stop_reason: Some(stop_reason), + usage: Some(usage), + error: None, + }; + (frames, response) + } +} + +pub(super) struct Message; + +impl Message { + pub(super) fn user(text: &str) -> Value { + json!({ + "role": "user", + "content": [{ "type": "text", "text": text }] + }) + } + + pub(super) fn function_call( + call_id: &str, + function: &RecorderFunction, + arguments: Value, + model: &ModelFixtureV1, + ) -> Value { + json!({ + "role": "assistant", + "content": [{ + "type": "function_call", + "id": call_id, + "function_id": function.id(), + "arguments": arguments + }], + "stop_reason": "end", + "model": model.id, + "provider": model.provider + }) + } + + pub(super) fn function_result(call_id: &str, function: &RecorderFunction, text: &str) -> Value { + json!({ + "role": "function_result", + "function_call_id": call_id, + "function_id": function.id(), + "content": [{ "type": "text", "text": text }], + "details": function.target.response, + "is_error": false + }) + } +} + +pub(super) struct Tool; + +impl Tool { + pub(super) fn named(name: &str) -> Value { + json!({ "name": name }) + } +} + +fn recorder_config(target: RecorderTargetV1) -> RecorderConfigV1 { + RecorderConfigV1 { + target, + lifecycle: RecorderLifecycleV1 { + trigger_type: LifecycleTriggerType::TurnCompleted, + function_id: LifecycleFunctionId::Lifecycle, + }, + } +} + +fn usage(input: u64, output: u64) -> Usage { + Usage { + input: Some(input), + output: Some(output), + ..Default::default() + } +} + +fn exact(expected: Value) -> JsonMatcherV1 { + JsonMatcherV1::Exact { + expected, + normalize: None, + } +} + +fn assistant_message( + content: Vec, + stop_reason: StopReason, + usage: Option, + model: &ModelFixtureV1, + timestamp: i64, +) -> AssistantMessage { + AssistantMessage { + role: AssistantRoleTag::Assistant, + content, + stop_reason, + native_stop_reason: None, + error_message: None, + error_kind: None, + warnings: None, + usage, + model: model.id.clone(), + provider: model.provider.clone(), + timestamp, + } +} + +fn streamed_text_frames( + text: &str, + chunks: &[String], + usage: &Usage, + model: &ModelFixtureV1, + timestamp: i64, +) -> Vec { + let final_message = assistant_message( + vec![ContentBlock::Text { + text: text.to_string(), + }], + StopReason::End, + Some(usage.clone()), + model, + timestamp, + ); + if chunks.is_empty() { + return vec![AssistantMessageEvent::Done { + message: final_message, + }]; + } + let mut frames = vec![ + AssistantMessageEvent::Start { + partial: assistant_message(Vec::new(), StopReason::End, None, model, timestamp), + }, + AssistantMessageEvent::TextStart { + partial: assistant_message( + vec![ContentBlock::Text { + text: String::new(), + }], + StopReason::End, + None, + model, + timestamp, + ), + }, + ]; + frames.extend( + chunks + .iter() + .cloned() + .map(|delta| AssistantMessageEvent::TextDelta { + partial: None, + delta, + }), + ); + frames.extend([ + AssistantMessageEvent::TextEnd { + partial: assistant_message( + vec![ContentBlock::Text { + text: text.to_string(), + }], + StopReason::End, + None, + model, + timestamp, + ), + }, + AssistantMessageEvent::Usage { + usage: usage.clone(), + }, + AssistantMessageEvent::Stop { + stop_reason: StopReason::End, + error_message: None, + error_kind: None, + }, + AssistantMessageEvent::Done { + message: final_message, + }, + ]); + frames +} + +fn system_prompt(allowed_functions: &[String]) -> String { + let base = DEFAULT_SYSTEM_PROMPT + .strip_suffix('\n') + .unwrap_or(DEFAULT_SYSTEM_PROMPT); + let policy = if allowed_functions.is_empty() { + "Function dispatch is entirely disabled this turn — do not call any function.".to_string() + } else { + format!( + "Your dispatch policy allows ONLY these functions: {}. This narrowed-policy \ + instruction OVERRIDES the general discovery requirement for this turn: call the \ + listed target ids directly when the task already supplies their arguments. Anything \ + else — including discovery (engine::functions::list / ::info) unless listed above — \ + is denied. Do not probe: if the task genuinely needs an unlisted function or an \ + unknown contract, report that blocker and finish.", + allowed_functions.join(", ") + ) + }; + format!("{base}\n\nYour session id is {{{{session_id}}}}.\n{policy}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn streamed_response_has_one_terminal_frame() { + let model = Model::scripted("fixture-model"); + let (frames, response) = + Response::streamed_text("complete", ["com", "plete"], 2, 1).compile(&model, 1); + assert_eq!(frames.iter().filter(|frame| frame.is_terminal()).count(), 1); + assert!(frames.last().unwrap().is_terminal()); + assert_eq!(response.stop_reason, Some(StopReason::End)); + } + + #[test] + fn recorder_function_uses_one_contract_for_tool_and_target() { + let function = RecorderFunction::new("{{run_id}}::record", "Record value") + .request_schema(json!({ "type": "object" })) + .returns_text("recorded"); + let tool = function.tool(); + let recorder = Recorder::function(function.clone()); + assert_eq!(tool["name"], recorder.target.function_id); + assert_eq!( + tool["parameters"], + Value::Object(recorder.target.request_schema.clone()) + ); + assert_eq!(function.target.response, recorder.target.response); + } +} diff --git a/harness/evals/integration/src/scenarios/exactly_once_function.rs b/harness/evals/integration/src/scenarios/exactly_once_function.rs index 2ac4476e7..a1def368a 100644 --- a/harness/evals/integration/src/scenarios/exactly_once_function.rs +++ b/harness/evals/integration/src/scenarios/exactly_once_function.rs @@ -1,141 +1,84 @@ //! E2E-002 — the recorder runs exactly once and its result closes the turn. -use anyhow::ensure; use serde_json::json; -use super::support::{ - assistant_message, model, recorder, recorder_target, request_match, response, send, - system_prompt, usage, user_message, RequestProfile, MODEL_ID, PROVIDER_ID, +use super::dsl::{ + Generation, Message, Model, Recorder, RecorderFunction, Request, Response, Scenario, Send, }; use super::ScenarioDriver; use crate::fixtures::ScenarioFixture; -use crate::types::frames::{AssistantMessageEvent, ContentBlock, StopReason}; -use crate::types::scenario::{CompiledScenarioV1, DeadlinesV1}; -use crate::types::script::{RouterScriptV1, SchemaVersion1, ScriptedGenerationV1}; pub(super) fn scenario() -> ScenarioFixture { const ID: &str = "E2E-002"; const MESSAGE: &str = "Call the recorder once."; - const FUNCTION_ID: &str = "{{run_id}}::record"; + const CALL_ID: &str = "call-1"; - let model = model(); - let target = recorder_target(FUNCTION_ID); - let allowed_functions = vec![FUNCTION_ID.to_string()]; - let tools = json!([{ - "name": FUNCTION_ID, - "description": target.description, - "parameters": target.request_schema, - "execution_mode": "sequential" - }]); + let model = Model::scripted("fixture-model"); + let record = RecorderFunction::new( + "{{run_id}}::record", + "Record one integration fixture value.", + ) + .request_schema(json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + })) + .returns_text("recorded"); let arguments = json!({ "value": "expected" }); - let call_usage = usage(8, 4); - let final_usage = usage(18, 2); - let first_messages = vec![user_message(MESSAGE)]; - let function_call = assistant_message( - vec![ContentBlock::FunctionCall { - id: "call-1".to_string(), - function_id: FUNCTION_ID.to_string(), - arguments: arguments.clone(), - }], - StopReason::FunctionCall, - Some(call_usage.clone()), - &model, - 1, - ); - let mut second_messages = first_messages.clone(); - second_messages.extend([ - json!({ - "role": "assistant", - "content": [{ - "type": "function_call", - "id": "call-1", - "function_id": FUNCTION_ID, - "arguments": arguments - }], - "stop_reason": "end", - "model": MODEL_ID, - "provider": PROVIDER_ID - }), - json!({ - "role": "function_result", - "function_call_id": "call-1", - "function_id": FUNCTION_ID, - "content": [{ "type": "text", "text": "recorded" }], - "details": target.response, - "is_error": false - }), - ]); - let final_message = assistant_message( - vec![ContentBlock::Text { - text: "recorded once".to_string(), - }], - StopReason::End, - Some(final_usage.clone()), - &model, - 2, - ); - let generations = vec![ - ScriptedGenerationV1 { - ordinal: 1, - match_: request_match(1, &model, &first_messages, &tools, RequestProfile::Direct), - frames: vec![AssistantMessageEvent::Done { - message: function_call, - }], - response: response(StopReason::FunctionCall, call_usage, &model), - }, - ScriptedGenerationV1 { - ordinal: 2, - match_: request_match(2, &model, &second_messages, &tools, RequestProfile::Direct), - frames: vec![AssistantMessageEvent::Done { - message: final_message, - }], - response: response(StopReason::End, final_usage, &model), - }, - ]; - - ScenarioFixture { - slug: "exactly-once-function".to_string(), - driver: ScenarioDriver::Direct, - scenario: CompiledScenarioV1 { - schema_version: SchemaVersion1::V1, - id: ID.to_string(), - description: "The recorder runs exactly once.".to_string(), - send: send(ID, MESSAGE, &model, &allowed_functions), - recorder: recorder(target), - deadlines: DeadlinesV1::default(), - }, - script: RouterScriptV1 { - schema_version: SchemaVersion1::V1, - scenario_id: ID.to_string(), - model, - generations, - }, - system_prompt_template: system_prompt(&allowed_functions), - verify: |run| { - let texts = run.assistant_texts(); - ensure!( - texts == ["recorded once"], - "assistant texts {texts:?} != [\"recorded once\"]" - ); - let calls = run.calls("record"); - ensure!( - calls.len() == 1, - "record ran {} times, not exactly once", - calls.len() - ); - let payload = &calls[0].payload; - ensure!( - payload == &json!({ "value": "expected" }), - "record payload {payload} != {{\"value\":\"expected\"}}" - ); - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" - ); - Ok(()) - }, - } + Scenario::new( + ID, + "exactly-once-function", + "The recorder runs exactly once.", + ScenarioDriver::Direct, + model.clone(), + ) + .send( + Send::message(MESSAGE) + .idempotency_key("{{run_id}}:e2e-002") + .allow_function(&record), + ) + .recorder(Recorder::function(record.clone())) + .generation( + Generation::new(1) + .expect( + Request::new() + .turn_request() + .system_prompt_sha256("{{system_prompt_sha256}}") + .messages_exact([Message::user(MESSAGE)]) + .tools_exact([record.tool()]), + ) + .respond(Response::function_call( + CALL_ID, + &record, + arguments.clone(), + 8, + 4, + )), + ) + .generation( + Generation::new(2) + .expect( + Request::new() + .turn_request() + .system_prompt_sha256("{{system_prompt_sha256}}") + .messages_exact([ + Message::user(MESSAGE), + Message::function_call(CALL_ID, &record, arguments.clone(), &model), + Message::function_result(CALL_ID, &record, "recorded"), + ]) + .tools_exact([record.tool()]), + ) + .respond(Response::text("recorded once", 18, 2)), + ) + .verify(|run| { + run.expect_assistant_texts(["recorded once"])?; + run.expect_function_calls("record", 1)?; + run.expect_call_payload("record", json!({ "value": "expected" }))?; + run.expect_no_duplicate_messages() + }) + .build() } #[cfg(test)] diff --git a/harness/evals/integration/src/scenarios/mod.rs b/harness/evals/integration/src/scenarios/mod.rs index cea35fd72..32fde7e44 100644 --- a/harness/evals/integration/src/scenarios/mod.rs +++ b/harness/evals/integration/src/scenarios/mod.rs @@ -1,9 +1,9 @@ //! The three checked-in integration fixtures. mod console_streamed_text; +mod dsl; mod exactly_once_function; mod streamed_text; -mod support; use crate::evidence_data::RunEvidence; use crate::fixtures::ScenarioFixture; diff --git a/harness/evals/integration/src/scenarios/streamed_text.rs b/harness/evals/integration/src/scenarios/streamed_text.rs index 9c3403f19..0958b8d38 100644 --- a/harness/evals/integration/src/scenarios/streamed_text.rs +++ b/harness/evals/integration/src/scenarios/streamed_text.rs @@ -1,74 +1,55 @@ //! E2E-001 — streamed text reaches durable completion. -use anyhow::ensure; -use serde_json::json; - -use super::support::{ - model, request_match, response, send, streamed_text_frames, synthetic_recorder, system_prompt, - usage, user_message, RequestProfile, -}; +use super::dsl::{Generation, Message, Model, Recorder, Request, Response, Scenario, Send}; use super::ScenarioDriver; use crate::fixtures::ScenarioFixture; -use crate::types::frames::StopReason; -use crate::types::scenario::{CompiledScenarioV1, DeadlinesV1}; -use crate::types::script::{RouterScriptV1, SchemaVersion1, ScriptedGenerationV1}; pub(super) fn scenario() -> ScenarioFixture { const ID: &str = "E2E-001"; const MESSAGE: &str = "Return the fixture phrase."; const TEXT: &str = "fixture complete"; - let model = model(); - let usage = usage(8, 2); - let allowed_functions = Vec::new(); - let messages = vec![user_message(MESSAGE)]; - let generation = ScriptedGenerationV1 { - ordinal: 1, - match_: request_match(1, &model, &messages, &json!([]), RequestProfile::Direct), - frames: streamed_text_frames(TEXT, &["fixture ", "complete"], &usage, &model), - response: response(StopReason::End, usage, &model), - }; - - ScenarioFixture { - slug: "streamed-text".to_string(), - driver: ScenarioDriver::Direct, - scenario: CompiledScenarioV1 { - schema_version: SchemaVersion1::V1, - id: ID.to_string(), - description: - "Streamed text reaches durable completion through the real queue and turn loop." - .to_string(), - send: send(ID, MESSAGE, &model, &allowed_functions), - recorder: synthetic_recorder(), - deadlines: DeadlinesV1::default(), - }, - script: RouterScriptV1 { - schema_version: SchemaVersion1::V1, - scenario_id: ID.to_string(), - model, - generations: vec![generation], - }, - system_prompt_template: system_prompt(&allowed_functions), - verify: |run| { - let texts = run.assistant_texts(); - ensure!(texts == [TEXT], "assistant texts {texts:?} != [\"{TEXT}\"]"); - let counts = run.message_counts(); - ensure!( - counts == (1, 1, 0), - "message counts (user, assistant, function_result) {counts:?} != (1, 1, 0)" - ); - ensure!( - !run.has_duplicate_messages(), - "transcript contains duplicate entry ids" - ); - Ok(()) - }, - } + Scenario::new( + ID, + "streamed-text", + "Streamed text reaches durable completion through the real queue and turn loop.", + ScenarioDriver::Direct, + Model::scripted("fixture-model"), + ) + .send( + Send::message(MESSAGE) + .idempotency_key("{{run_id}}:e2e-001") + .without_functions(), + ) + .recorder(Recorder::lifecycle_only()) + .generation( + Generation::new(1) + .expect( + Request::new() + .turn_request() + .system_prompt_sha256("{{system_prompt_sha256}}") + .messages_exact([Message::user(MESSAGE)]) + .without_tools(), + ) + .respond(Response::streamed_text( + TEXT, + ["fixture ", "complete"], + 8, + 2, + )), + ) + .verify(|run| { + run.expect_assistant_texts([TEXT])?; + run.expect_message_counts(1, 1, 0)?; + run.expect_no_duplicate_messages() + }) + .build() } #[cfg(test)] mod tests { use super::*; + use crate::types::frames::StopReason; #[test] fn stream_has_one_terminal_frame_and_matching_response() { diff --git a/harness/evals/integration/src/scenarios/support.rs b/harness/evals/integration/src/scenarios/support.rs deleted file mode 100644 index 0b1347384..000000000 --- a/harness/evals/integration/src/scenarios/support.rs +++ /dev/null @@ -1,314 +0,0 @@ -//! Shared wire constructors used by the checked-in scenarios. - -use serde_json::{json, Value}; - -use crate::types::frames::{ - AssistantMessage, AssistantMessageEvent, AssistantRoleTag, ContentBlock, RouterChatResponse, - StopReason, Usage, -}; -use crate::types::recorder::{ - LifecycleFunctionId, LifecycleTriggerType, RecorderConfigV1, RecorderLifecycleV1, - RecorderTargetV1, -}; -use crate::types::scenario::{ - CompiledFunctionExposureV1, CompiledFunctionPolicyV1, CompiledSendOptionsV1, CompiledSendV1, -}; -use crate::types::script::{ - GenerationMatchV1, JsonMatcherV1, JsonNormalizerV1, ModelFixtureV1, NormalizerOperation, -}; - -const DEFAULT_SYSTEM_PROMPT: &str = include_str!("../../../../prompts/default.txt"); -pub(super) const MODEL_ID: &str = "fixture-model"; -pub(super) const PROVIDER_ID: &str = "scripted"; - -#[derive(Debug, Clone, Copy)] -pub(super) enum RequestProfile { - Direct, - Console, -} - -pub(super) fn model() -> ModelFixtureV1 { - ModelFixtureV1 { - id: MODEL_ID.to_string(), - provider: PROVIDER_ID.to_string(), - context_window: 32_768, - max_output_tokens: 4_096, - supports_thinking: Some(false), - supports_xhigh: None, - supports_tools: Some(true), - supports_vision: Some(false), - supports_cache: Some(false), - supports_structured_output: Some(true), - } -} - -pub(super) fn send( - scenario_id: &str, - message: &str, - model: &ModelFixtureV1, - allowed_functions: &[String], -) -> CompiledSendV1 { - CompiledSendV1 { - session_id: "{{session_id}}".to_string(), - message: message.to_string(), - model: model.id.clone(), - provider: model.provider.clone(), - idempotency_key: format!("{{{{run_id}}}}:{}", scenario_id.to_ascii_lowercase()), - options: CompiledSendOptionsV1 { - functions: CompiledFunctionPolicyV1 { - allow: allowed_functions.to_vec(), - deny: Vec::new(), - expose: CompiledFunctionExposureV1::Native, - }, - }, - } -} - -pub(super) fn request_match( - ordinal: u64, - model: &ModelFixtureV1, - messages: &[Value], - tools: &Value, - profile: RequestProfile, -) -> GenerationMatchV1 { - let normalizers = (0..messages.len()) - .map(|index| JsonNormalizerV1 { - pointer: format!("/{index}/timestamp"), - operation: NormalizerOperation::Delete, - }) - .collect(); - let (system_prompt, tools) = match profile { - RequestProfile::Direct => ( - JsonMatcherV1::Sha256 { - expected: "{{system_prompt_sha256}}".to_string(), - }, - exact(tools.clone()), - ), - RequestProfile::Console => ( - JsonMatcherV1::Regex { - pattern: "agent_trigger".to_string(), - }, - JsonMatcherV1::Subset { - expected: json!([{ "name": "agent_trigger" }]), - normalize: None, - }, - ), - }; - GenerationMatchV1 { - writer_ref: JsonMatcherV1::Subset { - expected: json!({ "direction": "write" }), - normalize: None, - }, - request_id: JsonMatcherV1::Regex { - pattern: if ordinal == 1 { - "^t_[0-9a-f]{32}:[0-9]+$".to_string() - } else { - format!("^t_[0-9a-f]{{32}}:{}$", ordinal - 1) - }, - }, - model: exact(json!(model.id)), - provider: exact(json!(model.provider)), - system_prompt, - messages: JsonMatcherV1::Exact { - expected: Value::Array(messages.to_vec()), - normalize: Some(normalizers), - }, - tools, - response_format: JsonMatcherV1::Absent, - thinking_level: JsonMatcherV1::Absent, - max_output_tokens: JsonMatcherV1::Absent, - provider_options: JsonMatcherV1::Absent, - metadata: JsonMatcherV1::Absent, - } -} - -fn exact(expected: Value) -> JsonMatcherV1 { - JsonMatcherV1::Exact { - expected, - normalize: None, - } -} - -pub(super) fn user_message(message: &str) -> Value { - json!({ - "role": "user", - "content": [{ "type": "text", "text": message }] - }) -} - -pub(super) fn assistant_message( - content: Vec, - stop_reason: StopReason, - usage: Option, - model: &ModelFixtureV1, - timestamp: i64, -) -> AssistantMessage { - AssistantMessage { - role: AssistantRoleTag::Assistant, - content, - stop_reason, - native_stop_reason: None, - error_message: None, - error_kind: None, - warnings: None, - usage, - model: model.id.clone(), - provider: model.provider.clone(), - timestamp, - } -} - -pub(super) fn streamed_text_frames( - text: &str, - chunks: &[&str], - usage: &Usage, - model: &ModelFixtureV1, -) -> Vec { - let message = assistant_message( - vec![ContentBlock::Text { - text: text.to_string(), - }], - StopReason::End, - Some(usage.clone()), - model, - 1, - ); - if chunks.is_empty() { - return vec![AssistantMessageEvent::Done { message }]; - } - - let mut frames = vec![ - AssistantMessageEvent::Start { - partial: assistant_message(Vec::new(), StopReason::End, None, model, 1), - }, - AssistantMessageEvent::TextStart { - partial: assistant_message( - vec![ContentBlock::Text { - text: String::new(), - }], - StopReason::End, - None, - model, - 1, - ), - }, - ]; - frames.extend(chunks.iter().map(|chunk| AssistantMessageEvent::TextDelta { - partial: None, - delta: (*chunk).to_string(), - })); - frames.extend([ - AssistantMessageEvent::TextEnd { - partial: assistant_message( - vec![ContentBlock::Text { - text: text.to_string(), - }], - StopReason::End, - None, - model, - 1, - ), - }, - AssistantMessageEvent::Usage { - usage: usage.clone(), - }, - AssistantMessageEvent::Stop { - stop_reason: StopReason::End, - error_message: None, - error_kind: None, - }, - AssistantMessageEvent::Done { message }, - ]); - frames -} - -pub(super) fn response( - stop_reason: StopReason, - usage: Usage, - model: &ModelFixtureV1, -) -> RouterChatResponse { - RouterChatResponse { - ok: true, - provider: model.provider.clone(), - model: model.id.clone(), - stop_reason: Some(stop_reason), - usage: Some(usage), - error: None, - } -} - -pub(super) fn usage(input: u64, output: u64) -> Usage { - Usage { - input: Some(input), - output: Some(output), - ..Default::default() - } -} - -pub(super) fn recorder(target: RecorderTargetV1) -> RecorderConfigV1 { - RecorderConfigV1 { - target, - lifecycle: RecorderLifecycleV1 { - trigger_type: LifecycleTriggerType::TurnCompleted, - function_id: LifecycleFunctionId::Lifecycle, - }, - } -} - -pub(super) fn synthetic_recorder() -> RecorderConfigV1 { - recorder(RecorderTargetV1 { - function_id: "{{run_id}}::unused".to_string(), - description: "Synthetic integration target; must never be called.".to_string(), - request_schema: json!({ - "type": "object", - "additionalProperties": false - }) - .as_object() - .expect("object") - .clone(), - response: json!({ - "content": [{ "type": "text", "text": "unused" }], - "is_error": false - }), - }) -} - -pub(super) fn recorder_target(function_id: &str) -> RecorderTargetV1 { - RecorderTargetV1 { - function_id: function_id.to_string(), - description: "Record one integration fixture value.".to_string(), - request_schema: json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }) - .as_object() - .expect("object") - .clone(), - response: json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - } -} - -pub(super) fn system_prompt(allowed_functions: &[String]) -> String { - let base = DEFAULT_SYSTEM_PROMPT - .strip_suffix('\n') - .unwrap_or(DEFAULT_SYSTEM_PROMPT); - let policy = if allowed_functions.is_empty() { - "Function dispatch is entirely disabled this turn — do not call any function.".to_string() - } else { - format!( - "Your dispatch policy allows ONLY these functions: {}. This narrowed-policy \ - instruction OVERRIDES the general discovery requirement for this turn: call the \ - listed target ids directly when the task already supplies their arguments. Anything \ - else — including discovery (engine::functions::list / ::info) unless listed above — \ - is denied. Do not probe: if the task genuinely needs an unlisted function or an \ - unknown contract, report that blocker and finish.", - allowed_functions.join(", ") - ) - }; - format!("{base}\n\nYour session id is {{{{session_id}}}}.\n{policy}") -} From 45e8225168388fd341196c14360be79d970c63a6 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Wed, 22 Jul 2026 09:07:06 -0300 Subject: [PATCH 4/5] (MOT-4107) refactor(harness): move integration runner into test workspace --- .github/workflows/ci.yml | 7 +- harness/Cargo.lock | 47 + harness/Cargo.toml | 7 + harness/Makefile | 24 +- harness/evals/integration/Cargo.lock | 2586 ----------------- .../integration => tests/e2e}/Cargo.toml | 9 - .../integration => tests/e2e}/README.md | 6 +- .../integration => tests/e2e}/engine.lock | 0 .../e2e}/src/artifacts.rs | 0 .../e2e}/src/artifacts/sink.rs | 0 .../e2e}/src/artifacts/tests.rs | 0 .../e2e}/src/canonical.rs | 0 .../integration => tests/e2e}/src/client.rs | 0 .../integration => tests/e2e}/src/deadline.rs | 0 .../e2e}/src/discovery.rs | 0 .../e2e}/src/evidence_data.rs | 0 .../integration => tests/e2e}/src/expand.rs | 0 .../e2e}/src/expand/tokens.rs | 0 .../integration => tests/e2e}/src/fixtures.rs | 0 .../e2e}/src/fixtures/discovery.rs | 0 .../e2e}/src/fixtures/loading.rs | 0 .../e2e}/src/fixtures/tests.rs | 0 .../integration => tests/e2e}/src/lib.rs | 0 .../integration => tests/e2e}/src/main.rs | 0 .../integration => tests/e2e}/src/matcher.rs | 0 .../integration => tests/e2e}/src/process.rs | 0 .../e2e}/src/process/child.rs | 0 .../e2e}/src/process/spec.rs | 0 .../e2e}/src/process/supervisor.rs | 0 .../e2e}/src/process/tests.rs | 0 .../integration => tests/e2e}/src/recorder.rs | 0 .../e2e}/src/recorder/service.rs | 0 .../e2e}/src/recorder/store.rs | 0 .../e2e}/src/recorder/tests.rs | 0 .../integration => tests/e2e}/src/runtime.rs | 0 .../integration => tests/e2e}/src/scenario.rs | 0 .../e2e}/src/scenario/floor.rs | 0 .../e2e}/src/scenario/phases/arm.rs | 0 .../e2e}/src/scenario/phases/completion.rs | 0 .../e2e}/src/scenario/phases/evidence.rs | 0 .../e2e}/src/scenario/phases/execution.rs | 0 .../e2e}/src/scenario/phases/mod.rs | 0 .../e2e}/src/scenario/playground.rs | 0 .../e2e}/src/scenario/report.rs | 0 .../e2e}/src/scenario/runner.rs | 0 .../e2e}/src/scenario/state.rs | 0 .../e2e}/src/scenario/tests.rs | 0 .../src/scenarios/console_streamed_text.rs | 0 .../e2e}/src/scenarios/dsl.rs | 0 .../src/scenarios/exactly_once_function.rs | 0 .../e2e}/src/scenarios/mod.rs | 0 .../e2e}/src/scenarios/streamed_text.rs | 0 .../e2e}/src/scripted_router.rs | 0 .../integration => tests/e2e}/src/services.rs | 0 .../integration => tests/e2e}/src/stack.rs | 0 .../e2e}/src/stack/bins.rs | 0 .../e2e}/src/stack/config.rs | 0 .../e2e}/src/stack/layout.rs | 0 .../e2e}/src/stack/manifest.rs | 0 .../e2e}/src/stack/supervisor.rs | 0 .../e2e}/src/stack/tests.rs | 0 .../e2e}/src/types/frames.rs | 0 .../e2e}/src/types/mod.rs | 0 .../e2e}/src/types/recorder.rs | 0 .../e2e}/src/types/scenario.rs | 0 .../e2e}/src/types/scenario/compiled.rs | 0 .../e2e}/src/types/scenario/result.rs | 0 .../e2e}/src/types/script.rs | 0 .../e2e}/tests/determinism.rs | 0 .../e2e}/tests/scenario_compilation.rs | 0 .../e2e}/tests/schemas.rs | 0 .../e2e}/tests/supervisor.rs | 0 72 files changed, 72 insertions(+), 2614 deletions(-) delete mode 100644 harness/evals/integration/Cargo.lock rename harness/{evals/integration => tests/e2e}/Cargo.toml (74%) rename harness/{evals/integration => tests/e2e}/README.md (96%) rename harness/{evals/integration => tests/e2e}/engine.lock (100%) rename harness/{evals/integration => tests/e2e}/src/artifacts.rs (100%) rename harness/{evals/integration => tests/e2e}/src/artifacts/sink.rs (100%) rename harness/{evals/integration => tests/e2e}/src/artifacts/tests.rs (100%) rename harness/{evals/integration => tests/e2e}/src/canonical.rs (100%) rename harness/{evals/integration => tests/e2e}/src/client.rs (100%) rename harness/{evals/integration => tests/e2e}/src/deadline.rs (100%) rename harness/{evals/integration => tests/e2e}/src/discovery.rs (100%) rename harness/{evals/integration => tests/e2e}/src/evidence_data.rs (100%) rename harness/{evals/integration => tests/e2e}/src/expand.rs (100%) rename harness/{evals/integration => tests/e2e}/src/expand/tokens.rs (100%) rename harness/{evals/integration => tests/e2e}/src/fixtures.rs (100%) rename harness/{evals/integration => tests/e2e}/src/fixtures/discovery.rs (100%) rename harness/{evals/integration => tests/e2e}/src/fixtures/loading.rs (100%) rename harness/{evals/integration => tests/e2e}/src/fixtures/tests.rs (100%) rename harness/{evals/integration => tests/e2e}/src/lib.rs (100%) rename harness/{evals/integration => tests/e2e}/src/main.rs (100%) rename harness/{evals/integration => tests/e2e}/src/matcher.rs (100%) rename harness/{evals/integration => tests/e2e}/src/process.rs (100%) rename harness/{evals/integration => tests/e2e}/src/process/child.rs (100%) rename harness/{evals/integration => tests/e2e}/src/process/spec.rs (100%) rename harness/{evals/integration => tests/e2e}/src/process/supervisor.rs (100%) rename harness/{evals/integration => tests/e2e}/src/process/tests.rs (100%) rename harness/{evals/integration => tests/e2e}/src/recorder.rs (100%) rename harness/{evals/integration => tests/e2e}/src/recorder/service.rs (100%) rename harness/{evals/integration => tests/e2e}/src/recorder/store.rs (100%) rename harness/{evals/integration => tests/e2e}/src/recorder/tests.rs (100%) rename harness/{evals/integration => tests/e2e}/src/runtime.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/floor.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/phases/arm.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/phases/completion.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/phases/evidence.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/phases/execution.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/phases/mod.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/playground.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/report.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/runner.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/state.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenario/tests.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenarios/console_streamed_text.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenarios/dsl.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenarios/exactly_once_function.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenarios/mod.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scenarios/streamed_text.rs (100%) rename harness/{evals/integration => tests/e2e}/src/scripted_router.rs (100%) rename harness/{evals/integration => tests/e2e}/src/services.rs (100%) rename harness/{evals/integration => tests/e2e}/src/stack.rs (100%) rename harness/{evals/integration => tests/e2e}/src/stack/bins.rs (100%) rename harness/{evals/integration => tests/e2e}/src/stack/config.rs (100%) rename harness/{evals/integration => tests/e2e}/src/stack/layout.rs (100%) rename harness/{evals/integration => tests/e2e}/src/stack/manifest.rs (100%) rename harness/{evals/integration => tests/e2e}/src/stack/supervisor.rs (100%) rename harness/{evals/integration => tests/e2e}/src/stack/tests.rs (100%) rename harness/{evals/integration => tests/e2e}/src/types/frames.rs (100%) rename harness/{evals/integration => tests/e2e}/src/types/mod.rs (100%) rename harness/{evals/integration => tests/e2e}/src/types/recorder.rs (100%) rename harness/{evals/integration => tests/e2e}/src/types/scenario.rs (100%) rename harness/{evals/integration => tests/e2e}/src/types/scenario/compiled.rs (100%) rename harness/{evals/integration => tests/e2e}/src/types/scenario/result.rs (100%) rename harness/{evals/integration => tests/e2e}/src/types/script.rs (100%) rename harness/{evals/integration => tests/e2e}/tests/determinism.rs (100%) rename harness/{evals/integration => tests/e2e}/tests/scenario_compilation.rs (100%) rename harness/{evals/integration => tests/e2e}/tests/schemas.rs (100%) rename harness/{evals/integration => tests/e2e}/tests/supervisor.rs (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73942324d..217a5b810 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -387,7 +387,7 @@ jobs: # ────────────────────────────────────────────────────────────── # Harness integration E2E (non-required): deterministic public-path # regression scenarios against the engine pinned in - # harness/evals/integration/engine.lock (see that crate's README; the + # harness/tests/e2e/engine.lock (see that crate's README; the # architecture spec lives in the iii repo). Runs on EVERY PR — the # suite is the harness stack's public-contract gate, and engine/worker # builds are cached. The engine is BUILT from the pinned source, never @@ -404,7 +404,7 @@ jobs: id: lock run: | set -euo pipefail - lock=harness/evals/integration/engine.lock + lock=harness/tests/e2e/engine.lock read_field() { local field=$1 values values=$(sed -n "s/^${field} = \"\\([^\"]*\\)\"$/\\1/p" "$lock") @@ -484,10 +484,9 @@ jobs: session-manager -> target context-manager -> target iii-directory -> target - harness/evals/integration -> target - name: Integration crate unit tests - run: cargo test --manifest-path harness/evals/integration/Cargo.toml + run: cargo test --manifest-path harness/Cargo.toml -p harness-integration - name: Validate integration scenarios run: make -C harness integration-validate diff --git a/harness/Cargo.lock b/harness/Cargo.lock index 1443468e3..b2fb83a7d 100644 --- a/harness/Cargo.lock +++ b/harness/Cargo.lock @@ -523,6 +523,30 @@ dependencies = [ "uuid", ] +[[package]] +name = "harness-integration" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "iii-sdk", + "jsonschema", + "nix", + "regex", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "sha2", + "tempfile", + "thiserror", + "time", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -959,6 +983,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "8.0.0" @@ -1661,6 +1697,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" diff --git a/harness/Cargo.toml b/harness/Cargo.toml index 6485c410e..773e10e9b 100644 --- a/harness/Cargo.toml +++ b/harness/Cargo.toml @@ -1,4 +1,6 @@ [workspace] +members = ["tests/e2e"] +resolver = "2" [package] name = "harness" @@ -44,3 +46,8 @@ jsonschema = { version = "0.18", default-features = false } serde_json = "1" tempfile = "3" tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "signal", "time"] } + +# The E2E runner digests every spawned binary before boot. Optimizing sha2 in +# development avoids a long stall when hashing large debug worker binaries. +[profile.dev.package.sha2] +opt-level = 3 diff --git a/harness/Makefile b/harness/Makefile index f025781b6..9c8b89e3e 100644 --- a/harness/Makefile +++ b/harness/Makefile @@ -61,7 +61,7 @@ help: ' make build [workers...] cargo build stack or selected workers' \ ' make cargo-clean [workers...] cargo clean stack or selected workers' \ ' make install-local [workers...] build and symlink into ~/.iii/workers' \ - ' make integration-e2e III_BIN= run the harness integration scenarios (evals/integration)' \ + ' make integration-e2e III_BIN= run the harness integration scenarios (tests/e2e)' \ ' make integration-playground III_BIN= open the Console against an isolated integration stack' \ ' make integration-validate validate every integration scenario' @@ -310,12 +310,12 @@ where: echo " no tmux session named $(TMUX_SESSION)" # --------------------------------------------------------------------------- -# Harness integration E2E (see harness/evals/integration/README.md) +# Harness integration E2E (see harness/tests/e2e/README.md) # -# Builds the subject stack + the standalone runner (evals/integration, its own -# workspace) and runs every scenario against the pinned engine binary. The +# Builds the subject stack + the harness workspace's E2E runner and runs every +# scenario against the pinned engine binary. The # engine is never downloaded here: pass III_BIN (CI builds it from -# evals/integration/engine.lock; locally see that file for the pinned source). +# tests/e2e/engine.lock; locally see that file for the pinned source). INTEGRATION_WORKERS := queue iii-directory session-manager context-manager INTEGRATION_PROFILE ?= release INTEGRATION_FLAG := $(if $(filter release,$(INTEGRATION_PROFILE)),--release,) @@ -327,7 +327,7 @@ INTEGRATION_PLAYGROUND_ARTIFACTS ?= $(REPO_ROOT)/target/console-e2e integration-e2e: @if [ -z "$(III_BIN)" ]; then \ echo "III_BIN is required: path to the pinned iii engine binary."; \ - echo "Pinned source: harness/evals/integration/engine.lock"; \ + echo "Pinned source: harness/tests/e2e/engine.lock"; \ exit 3; \ fi @for w in $(INTEGRATION_WORKERS) harness; do \ @@ -335,8 +335,8 @@ integration-e2e: cargo build $(INTEGRATION_FLAG) --manifest-path "$(REPO_ROOT)/$$w/Cargo.toml" || exit 1; \ done @echo "building harness-integration ($(INTEGRATION_PROFILE))" - @cargo build $(INTEGRATION_FLAG) --manifest-path "$(MAKEFILE_DIR)evals/integration/Cargo.toml" - @"$(MAKEFILE_DIR)evals/integration/target/$(INTEGRATION_PROFILE)/harness-integration" \ + @cargo build $(INTEGRATION_FLAG) --manifest-path "$(MAKEFILE_DIR)Cargo.toml" -p harness-integration + @"$(MAKEFILE_DIR)target/$(INTEGRATION_PROFILE)/harness-integration" \ run \ --engine-bin "$(III_BIN)" \ --harness-bin "$(REPO_ROOT)/harness/target/$(INTEGRATION_PROFILE)/harness" \ @@ -350,7 +350,7 @@ integration-e2e: integration-playground: @if [ -z "$(III_BIN)" ]; then \ echo "III_BIN is required: path to the pinned iii engine binary."; \ - echo "Pinned source: harness/evals/integration/engine.lock"; \ + echo "Pinned source: harness/tests/e2e/engine.lock"; \ exit 3; \ fi @for w in $(INTEGRATION_WORKERS) harness console; do \ @@ -358,8 +358,8 @@ integration-playground: cargo build $(INTEGRATION_FLAG) --manifest-path "$(REPO_ROOT)/$$w/Cargo.toml" || exit 1; \ done @echo "building harness-integration ($(INTEGRATION_PROFILE))" - @cargo build $(INTEGRATION_FLAG) --manifest-path "$(MAKEFILE_DIR)evals/integration/Cargo.toml" - @"$(MAKEFILE_DIR)evals/integration/target/$(INTEGRATION_PROFILE)/harness-integration" \ + @cargo build $(INTEGRATION_FLAG) --manifest-path "$(MAKEFILE_DIR)Cargo.toml" -p harness-integration + @"$(MAKEFILE_DIR)target/$(INTEGRATION_PROFILE)/harness-integration" \ playground \ --engine-bin "$(III_BIN)" \ --harness-bin "$(REPO_ROOT)/harness/target/$(INTEGRATION_PROFILE)/harness" \ @@ -372,7 +372,7 @@ integration-playground: --artifacts-dir "$(INTEGRATION_PLAYGROUND_ARTIFACTS)" integration-validate: - @cargo run --quiet --manifest-path "$(MAKEFILE_DIR)evals/integration/Cargo.toml" -- \ + @cargo run --quiet --manifest-path "$(MAKEFILE_DIR)Cargo.toml" -p harness-integration -- \ validate --scenario all $(STACK): diff --git a/harness/evals/integration/Cargo.lock b/harness/evals/integration/Cargo.lock deleted file mode 100644 index f57d10452..000000000 --- a/harness/evals/integration/Cargo.lock +++ /dev/null @@ -1,2586 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "serde", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - -[[package]] -name = "anyhow" -version = "1.0.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" - -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - -[[package]] -name = "bitflags" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "bytecount" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "cc" -version = "1.2.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - -[[package]] -name = "clap" -version = "4.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "data-encoding" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "displaydoc" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "fancy-regex" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "fastrand" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fraction" -version = "0.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" -dependencies = [ - "lazy_static", - "num", -] - -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", -] - -[[package]] -name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-executor" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-macro" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - -[[package]] -name = "futures-task" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" - -[[package]] -name = "futures-util" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" -dependencies = [ - "futures-core", - "futures-macro", - "futures-sink", - "futures-task", - "pin-project-lite", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", - "wasm-bindgen", -] - -[[package]] -name = "harness-integration" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "iii-sdk", - "jsonschema", - "nix", - "regex", - "schemars", - "serde", - "serde_json", - "serde_yaml", - "sha2", - "tempfile", - "thiserror", - "time", - "tokio", - "tracing", - "tracing-subscriber", - "uuid", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hostname" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" -dependencies = [ - "cfg-if", - "libc", - "windows-link", -] - -[[package]] -name = "http" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "hyper" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "iii-helpers" -version = "0.21.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d84d5c149ae4404365a79feca28aa66f6a7dbed56423b4b8c4e2421e0b5add" -dependencies = [ - "futures-util", - "opentelemetry", - "opentelemetry-http", - "opentelemetry_sdk", - "reqwest", - "schemars", - "serde", - "serde_json", - "sysinfo", - "tokio", - "tokio-tungstenite", - "tracing", - "uuid", -] - -[[package]] -name = "iii-sdk" -version = "0.21.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dd060fddcc9153b0dd07c038a14cf172ce15ce1d4edb98155563ed55b2caba" -dependencies = [ - "async-trait", - "futures-util", - "hostname", - "iii-helpers", - "reqwest", - "schemars", - "serde", - "serde_json", - "thiserror", - "tokio", - "tokio-tungstenite", - "tracing", - "uuid", -] - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "ipnet" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "iso8601" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1082f0c48f143442a1ac6122f67e360ceee130b967af4d50996e5154a45df46" -dependencies = [ - "nom", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "js-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "jsonschema" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0f4bea31643be4c6a678e9aa4ae44f0db9e5609d5ca9dc9083d06eb3e9a27a" -dependencies = [ - "ahash", - "anyhow", - "base64", - "bytecount", - "fancy-regex", - "fraction", - "getrandom 0.2.17", - "iso8601", - "itoa", - "memchr", - "num-cmp", - "once_cell", - "parking_lot", - "percent-encoding", - "regex", - "serde", - "serde_json", - "time", - "url", - "uuid", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.186" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "mio" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "nix" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - -[[package]] -name = "ntapi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" -dependencies = [ - "winapi", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-cmp" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags", -] - -[[package]] -name = "objc2-io-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" -dependencies = [ - "libc", - "objc2-core-foundation", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "opentelemetry" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "pin-project-lite", - "thiserror", - "tracing", -] - -[[package]] -name = "opentelemetry-http" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" -dependencies = [ - "async-trait", - "bytes", - "http", - "opentelemetry", - "reqwest", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" -dependencies = [ - "futures-channel", - "futures-executor", - "futures-util", - "opentelemetry", - "percent-encoding", - "rand 0.9.5", - "thiserror", - "tokio", - "tokio-stream", -] - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quinn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" -dependencies = [ - "bytes", - "getrandom 0.4.3", - "lru-slab", - "rand 0.10.2", - "rand_pcg", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.61.2", -] - -[[package]] -name = "quote" -version = "1.0.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" -dependencies = [ - "rand_chacha", - "rand_core 0.9.5", -] - -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64", - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls" -version = "0.23.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pki-types" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "schemars" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" -dependencies = [ - "dyn-clone", - "schemars_derive", - "serde", - "serde_json", -] - -[[package]] -name = "schemars_derive" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_derive_internals" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.150" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha1" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "socket2" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sysinfo" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" -dependencies = [ - "libc", - "memchr", - "ntapi", - "objc2-core-foundation", - "objc2-io-kit", - "windows", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "time" -version = "0.3.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" - -[[package]] -name = "time-macros" -version = "0.2.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.52.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" -dependencies = [ - "bytes", - "libc", - "mio", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" -dependencies = [ - "futures-util", - "log", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tungstenite", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", - "url", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tungstenite" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" -dependencies = [ - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.9.5", - "rustls", - "rustls-pki-types", - "sha1", - "thiserror", - "utf-8", -] - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "uuid" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" -dependencies = [ - "getrandom 0.4.3", - "js-sys", - "serde_core", - "wasm-bindgen", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" -dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" -dependencies = [ - "windows-core", -] - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-future" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" -dependencies = [ - "windows-core", - "windows-link", - "windows-threading", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-numerics" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" -dependencies = [ - "windows-core", - "windows-link", -] - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows-threading" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/harness/evals/integration/Cargo.toml b/harness/tests/e2e/Cargo.toml similarity index 74% rename from harness/evals/integration/Cargo.toml rename to harness/tests/e2e/Cargo.toml index 233cb283d..e8702829d 100644 --- a/harness/evals/integration/Cargo.toml +++ b/harness/tests/e2e/Cargo.toml @@ -1,7 +1,3 @@ -# Standalone workspace: keeps this crate out of any parent workspace, per the -# repo-wide one-worker-one-workspace convention and the integration spec. -[workspace] - [package] name = "harness-integration" version = "0.1.0" @@ -39,8 +35,3 @@ nix = { version = "0.29", features = ["signal", "process"] } [dev-dependencies] tempfile = "3" jsonschema = { version = "0.18", default-features = false } - -# stack.json digests every spawned binary (spec: recorded before boot); -# debug-mode sha2 turns ~800MB of debug worker binaries into a ~40s stall. -[profile.dev.package.sha2] -opt-level = 3 diff --git a/harness/evals/integration/README.md b/harness/tests/e2e/README.md similarity index 96% rename from harness/evals/integration/README.md rename to harness/tests/e2e/README.md index 7d613624b..d2b09541c 100644 --- a/harness/evals/integration/README.md +++ b/harness/tests/e2e/README.md @@ -87,9 +87,9 @@ compiled send through the SDK or submits through the Console UI. ```bash make -C harness integration-validate -cargo test --manifest-path harness/evals/integration/Cargo.toml -cargo clippy --manifest-path harness/evals/integration/Cargo.toml \ - --all-targets -- -D warnings +cargo test --manifest-path harness/Cargo.toml -p harness-integration +cargo clippy --manifest-path harness/Cargo.toml \ + -p harness-integration --all-targets -- -D warnings ``` `validate --scenario all` checks exactly the three fixtures. `run --scenario diff --git a/harness/evals/integration/engine.lock b/harness/tests/e2e/engine.lock similarity index 100% rename from harness/evals/integration/engine.lock rename to harness/tests/e2e/engine.lock diff --git a/harness/evals/integration/src/artifacts.rs b/harness/tests/e2e/src/artifacts.rs similarity index 100% rename from harness/evals/integration/src/artifacts.rs rename to harness/tests/e2e/src/artifacts.rs diff --git a/harness/evals/integration/src/artifacts/sink.rs b/harness/tests/e2e/src/artifacts/sink.rs similarity index 100% rename from harness/evals/integration/src/artifacts/sink.rs rename to harness/tests/e2e/src/artifacts/sink.rs diff --git a/harness/evals/integration/src/artifacts/tests.rs b/harness/tests/e2e/src/artifacts/tests.rs similarity index 100% rename from harness/evals/integration/src/artifacts/tests.rs rename to harness/tests/e2e/src/artifacts/tests.rs diff --git a/harness/evals/integration/src/canonical.rs b/harness/tests/e2e/src/canonical.rs similarity index 100% rename from harness/evals/integration/src/canonical.rs rename to harness/tests/e2e/src/canonical.rs diff --git a/harness/evals/integration/src/client.rs b/harness/tests/e2e/src/client.rs similarity index 100% rename from harness/evals/integration/src/client.rs rename to harness/tests/e2e/src/client.rs diff --git a/harness/evals/integration/src/deadline.rs b/harness/tests/e2e/src/deadline.rs similarity index 100% rename from harness/evals/integration/src/deadline.rs rename to harness/tests/e2e/src/deadline.rs diff --git a/harness/evals/integration/src/discovery.rs b/harness/tests/e2e/src/discovery.rs similarity index 100% rename from harness/evals/integration/src/discovery.rs rename to harness/tests/e2e/src/discovery.rs diff --git a/harness/evals/integration/src/evidence_data.rs b/harness/tests/e2e/src/evidence_data.rs similarity index 100% rename from harness/evals/integration/src/evidence_data.rs rename to harness/tests/e2e/src/evidence_data.rs diff --git a/harness/evals/integration/src/expand.rs b/harness/tests/e2e/src/expand.rs similarity index 100% rename from harness/evals/integration/src/expand.rs rename to harness/tests/e2e/src/expand.rs diff --git a/harness/evals/integration/src/expand/tokens.rs b/harness/tests/e2e/src/expand/tokens.rs similarity index 100% rename from harness/evals/integration/src/expand/tokens.rs rename to harness/tests/e2e/src/expand/tokens.rs diff --git a/harness/evals/integration/src/fixtures.rs b/harness/tests/e2e/src/fixtures.rs similarity index 100% rename from harness/evals/integration/src/fixtures.rs rename to harness/tests/e2e/src/fixtures.rs diff --git a/harness/evals/integration/src/fixtures/discovery.rs b/harness/tests/e2e/src/fixtures/discovery.rs similarity index 100% rename from harness/evals/integration/src/fixtures/discovery.rs rename to harness/tests/e2e/src/fixtures/discovery.rs diff --git a/harness/evals/integration/src/fixtures/loading.rs b/harness/tests/e2e/src/fixtures/loading.rs similarity index 100% rename from harness/evals/integration/src/fixtures/loading.rs rename to harness/tests/e2e/src/fixtures/loading.rs diff --git a/harness/evals/integration/src/fixtures/tests.rs b/harness/tests/e2e/src/fixtures/tests.rs similarity index 100% rename from harness/evals/integration/src/fixtures/tests.rs rename to harness/tests/e2e/src/fixtures/tests.rs diff --git a/harness/evals/integration/src/lib.rs b/harness/tests/e2e/src/lib.rs similarity index 100% rename from harness/evals/integration/src/lib.rs rename to harness/tests/e2e/src/lib.rs diff --git a/harness/evals/integration/src/main.rs b/harness/tests/e2e/src/main.rs similarity index 100% rename from harness/evals/integration/src/main.rs rename to harness/tests/e2e/src/main.rs diff --git a/harness/evals/integration/src/matcher.rs b/harness/tests/e2e/src/matcher.rs similarity index 100% rename from harness/evals/integration/src/matcher.rs rename to harness/tests/e2e/src/matcher.rs diff --git a/harness/evals/integration/src/process.rs b/harness/tests/e2e/src/process.rs similarity index 100% rename from harness/evals/integration/src/process.rs rename to harness/tests/e2e/src/process.rs diff --git a/harness/evals/integration/src/process/child.rs b/harness/tests/e2e/src/process/child.rs similarity index 100% rename from harness/evals/integration/src/process/child.rs rename to harness/tests/e2e/src/process/child.rs diff --git a/harness/evals/integration/src/process/spec.rs b/harness/tests/e2e/src/process/spec.rs similarity index 100% rename from harness/evals/integration/src/process/spec.rs rename to harness/tests/e2e/src/process/spec.rs diff --git a/harness/evals/integration/src/process/supervisor.rs b/harness/tests/e2e/src/process/supervisor.rs similarity index 100% rename from harness/evals/integration/src/process/supervisor.rs rename to harness/tests/e2e/src/process/supervisor.rs diff --git a/harness/evals/integration/src/process/tests.rs b/harness/tests/e2e/src/process/tests.rs similarity index 100% rename from harness/evals/integration/src/process/tests.rs rename to harness/tests/e2e/src/process/tests.rs diff --git a/harness/evals/integration/src/recorder.rs b/harness/tests/e2e/src/recorder.rs similarity index 100% rename from harness/evals/integration/src/recorder.rs rename to harness/tests/e2e/src/recorder.rs diff --git a/harness/evals/integration/src/recorder/service.rs b/harness/tests/e2e/src/recorder/service.rs similarity index 100% rename from harness/evals/integration/src/recorder/service.rs rename to harness/tests/e2e/src/recorder/service.rs diff --git a/harness/evals/integration/src/recorder/store.rs b/harness/tests/e2e/src/recorder/store.rs similarity index 100% rename from harness/evals/integration/src/recorder/store.rs rename to harness/tests/e2e/src/recorder/store.rs diff --git a/harness/evals/integration/src/recorder/tests.rs b/harness/tests/e2e/src/recorder/tests.rs similarity index 100% rename from harness/evals/integration/src/recorder/tests.rs rename to harness/tests/e2e/src/recorder/tests.rs diff --git a/harness/evals/integration/src/runtime.rs b/harness/tests/e2e/src/runtime.rs similarity index 100% rename from harness/evals/integration/src/runtime.rs rename to harness/tests/e2e/src/runtime.rs diff --git a/harness/evals/integration/src/scenario.rs b/harness/tests/e2e/src/scenario.rs similarity index 100% rename from harness/evals/integration/src/scenario.rs rename to harness/tests/e2e/src/scenario.rs diff --git a/harness/evals/integration/src/scenario/floor.rs b/harness/tests/e2e/src/scenario/floor.rs similarity index 100% rename from harness/evals/integration/src/scenario/floor.rs rename to harness/tests/e2e/src/scenario/floor.rs diff --git a/harness/evals/integration/src/scenario/phases/arm.rs b/harness/tests/e2e/src/scenario/phases/arm.rs similarity index 100% rename from harness/evals/integration/src/scenario/phases/arm.rs rename to harness/tests/e2e/src/scenario/phases/arm.rs diff --git a/harness/evals/integration/src/scenario/phases/completion.rs b/harness/tests/e2e/src/scenario/phases/completion.rs similarity index 100% rename from harness/evals/integration/src/scenario/phases/completion.rs rename to harness/tests/e2e/src/scenario/phases/completion.rs diff --git a/harness/evals/integration/src/scenario/phases/evidence.rs b/harness/tests/e2e/src/scenario/phases/evidence.rs similarity index 100% rename from harness/evals/integration/src/scenario/phases/evidence.rs rename to harness/tests/e2e/src/scenario/phases/evidence.rs diff --git a/harness/evals/integration/src/scenario/phases/execution.rs b/harness/tests/e2e/src/scenario/phases/execution.rs similarity index 100% rename from harness/evals/integration/src/scenario/phases/execution.rs rename to harness/tests/e2e/src/scenario/phases/execution.rs diff --git a/harness/evals/integration/src/scenario/phases/mod.rs b/harness/tests/e2e/src/scenario/phases/mod.rs similarity index 100% rename from harness/evals/integration/src/scenario/phases/mod.rs rename to harness/tests/e2e/src/scenario/phases/mod.rs diff --git a/harness/evals/integration/src/scenario/playground.rs b/harness/tests/e2e/src/scenario/playground.rs similarity index 100% rename from harness/evals/integration/src/scenario/playground.rs rename to harness/tests/e2e/src/scenario/playground.rs diff --git a/harness/evals/integration/src/scenario/report.rs b/harness/tests/e2e/src/scenario/report.rs similarity index 100% rename from harness/evals/integration/src/scenario/report.rs rename to harness/tests/e2e/src/scenario/report.rs diff --git a/harness/evals/integration/src/scenario/runner.rs b/harness/tests/e2e/src/scenario/runner.rs similarity index 100% rename from harness/evals/integration/src/scenario/runner.rs rename to harness/tests/e2e/src/scenario/runner.rs diff --git a/harness/evals/integration/src/scenario/state.rs b/harness/tests/e2e/src/scenario/state.rs similarity index 100% rename from harness/evals/integration/src/scenario/state.rs rename to harness/tests/e2e/src/scenario/state.rs diff --git a/harness/evals/integration/src/scenario/tests.rs b/harness/tests/e2e/src/scenario/tests.rs similarity index 100% rename from harness/evals/integration/src/scenario/tests.rs rename to harness/tests/e2e/src/scenario/tests.rs diff --git a/harness/evals/integration/src/scenarios/console_streamed_text.rs b/harness/tests/e2e/src/scenarios/console_streamed_text.rs similarity index 100% rename from harness/evals/integration/src/scenarios/console_streamed_text.rs rename to harness/tests/e2e/src/scenarios/console_streamed_text.rs diff --git a/harness/evals/integration/src/scenarios/dsl.rs b/harness/tests/e2e/src/scenarios/dsl.rs similarity index 100% rename from harness/evals/integration/src/scenarios/dsl.rs rename to harness/tests/e2e/src/scenarios/dsl.rs diff --git a/harness/evals/integration/src/scenarios/exactly_once_function.rs b/harness/tests/e2e/src/scenarios/exactly_once_function.rs similarity index 100% rename from harness/evals/integration/src/scenarios/exactly_once_function.rs rename to harness/tests/e2e/src/scenarios/exactly_once_function.rs diff --git a/harness/evals/integration/src/scenarios/mod.rs b/harness/tests/e2e/src/scenarios/mod.rs similarity index 100% rename from harness/evals/integration/src/scenarios/mod.rs rename to harness/tests/e2e/src/scenarios/mod.rs diff --git a/harness/evals/integration/src/scenarios/streamed_text.rs b/harness/tests/e2e/src/scenarios/streamed_text.rs similarity index 100% rename from harness/evals/integration/src/scenarios/streamed_text.rs rename to harness/tests/e2e/src/scenarios/streamed_text.rs diff --git a/harness/evals/integration/src/scripted_router.rs b/harness/tests/e2e/src/scripted_router.rs similarity index 100% rename from harness/evals/integration/src/scripted_router.rs rename to harness/tests/e2e/src/scripted_router.rs diff --git a/harness/evals/integration/src/services.rs b/harness/tests/e2e/src/services.rs similarity index 100% rename from harness/evals/integration/src/services.rs rename to harness/tests/e2e/src/services.rs diff --git a/harness/evals/integration/src/stack.rs b/harness/tests/e2e/src/stack.rs similarity index 100% rename from harness/evals/integration/src/stack.rs rename to harness/tests/e2e/src/stack.rs diff --git a/harness/evals/integration/src/stack/bins.rs b/harness/tests/e2e/src/stack/bins.rs similarity index 100% rename from harness/evals/integration/src/stack/bins.rs rename to harness/tests/e2e/src/stack/bins.rs diff --git a/harness/evals/integration/src/stack/config.rs b/harness/tests/e2e/src/stack/config.rs similarity index 100% rename from harness/evals/integration/src/stack/config.rs rename to harness/tests/e2e/src/stack/config.rs diff --git a/harness/evals/integration/src/stack/layout.rs b/harness/tests/e2e/src/stack/layout.rs similarity index 100% rename from harness/evals/integration/src/stack/layout.rs rename to harness/tests/e2e/src/stack/layout.rs diff --git a/harness/evals/integration/src/stack/manifest.rs b/harness/tests/e2e/src/stack/manifest.rs similarity index 100% rename from harness/evals/integration/src/stack/manifest.rs rename to harness/tests/e2e/src/stack/manifest.rs diff --git a/harness/evals/integration/src/stack/supervisor.rs b/harness/tests/e2e/src/stack/supervisor.rs similarity index 100% rename from harness/evals/integration/src/stack/supervisor.rs rename to harness/tests/e2e/src/stack/supervisor.rs diff --git a/harness/evals/integration/src/stack/tests.rs b/harness/tests/e2e/src/stack/tests.rs similarity index 100% rename from harness/evals/integration/src/stack/tests.rs rename to harness/tests/e2e/src/stack/tests.rs diff --git a/harness/evals/integration/src/types/frames.rs b/harness/tests/e2e/src/types/frames.rs similarity index 100% rename from harness/evals/integration/src/types/frames.rs rename to harness/tests/e2e/src/types/frames.rs diff --git a/harness/evals/integration/src/types/mod.rs b/harness/tests/e2e/src/types/mod.rs similarity index 100% rename from harness/evals/integration/src/types/mod.rs rename to harness/tests/e2e/src/types/mod.rs diff --git a/harness/evals/integration/src/types/recorder.rs b/harness/tests/e2e/src/types/recorder.rs similarity index 100% rename from harness/evals/integration/src/types/recorder.rs rename to harness/tests/e2e/src/types/recorder.rs diff --git a/harness/evals/integration/src/types/scenario.rs b/harness/tests/e2e/src/types/scenario.rs similarity index 100% rename from harness/evals/integration/src/types/scenario.rs rename to harness/tests/e2e/src/types/scenario.rs diff --git a/harness/evals/integration/src/types/scenario/compiled.rs b/harness/tests/e2e/src/types/scenario/compiled.rs similarity index 100% rename from harness/evals/integration/src/types/scenario/compiled.rs rename to harness/tests/e2e/src/types/scenario/compiled.rs diff --git a/harness/evals/integration/src/types/scenario/result.rs b/harness/tests/e2e/src/types/scenario/result.rs similarity index 100% rename from harness/evals/integration/src/types/scenario/result.rs rename to harness/tests/e2e/src/types/scenario/result.rs diff --git a/harness/evals/integration/src/types/script.rs b/harness/tests/e2e/src/types/script.rs similarity index 100% rename from harness/evals/integration/src/types/script.rs rename to harness/tests/e2e/src/types/script.rs diff --git a/harness/evals/integration/tests/determinism.rs b/harness/tests/e2e/tests/determinism.rs similarity index 100% rename from harness/evals/integration/tests/determinism.rs rename to harness/tests/e2e/tests/determinism.rs diff --git a/harness/evals/integration/tests/scenario_compilation.rs b/harness/tests/e2e/tests/scenario_compilation.rs similarity index 100% rename from harness/evals/integration/tests/scenario_compilation.rs rename to harness/tests/e2e/tests/scenario_compilation.rs diff --git a/harness/evals/integration/tests/schemas.rs b/harness/tests/e2e/tests/schemas.rs similarity index 100% rename from harness/evals/integration/tests/schemas.rs rename to harness/tests/e2e/tests/schemas.rs diff --git a/harness/evals/integration/tests/supervisor.rs b/harness/tests/e2e/tests/supervisor.rs similarity index 100% rename from harness/evals/integration/tests/supervisor.rs rename to harness/tests/e2e/tests/supervisor.rs From d289f3150efcb24d26f238d9ca35c75be089f493 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Wed, 22 Jul 2026 09:27:32 -0300 Subject: [PATCH 5/5] (MOT-4107) test: stabilize shell and integration process tests --- harness/tests/e2e/tests/supervisor.rs | 4 +++- shell/tests/jobs_lifecycle.rs | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/harness/tests/e2e/tests/supervisor.rs b/harness/tests/e2e/tests/supervisor.rs index 2943dbce9..3d2c8854c 100644 --- a/harness/tests/e2e/tests/supervisor.rs +++ b/harness/tests/e2e/tests/supervisor.rs @@ -142,7 +142,9 @@ async fn children_see_only_the_environment_allowlist() { async fn wait_for_pid(path: &std::path::Path) -> u32 { for _ in 0..50 { if let Ok(raw) = std::fs::read_to_string(path) { - return raw.trim().parse().expect("numeric descendant pid"); + if let Ok(pid) = raw.trim().parse() { + return pid; + } } tokio::time::sleep(Duration::from_millis(20)).await; } diff --git a/shell/tests/jobs_lifecycle.rs b/shell/tests/jobs_lifecycle.rs index a8a9a6ee1..48ad413f6 100644 --- a/shell/tests/jobs_lifecycle.rs +++ b/shell/tests/jobs_lifecycle.rs @@ -1,8 +1,13 @@ //! Integration tests for the jobs lifecycle (`list_all`, `running_count`, -//! `remove_old`). Each test uses a unique ID prefix to avoid collisions in -//! the global `JOBS` map shared across the integration-test binary. +//! `remove_old`). Tests that mutate the global `JOBS` map are serialized: +//! unique IDs prevent replacement, but `remove_old` sweeps every job. + +use std::sync::LazyLock; use shell::jobs::{self, now_ms, JobHandle, JobRecord, JobStatus}; +use tokio::sync::Mutex; + +static JOBS_TEST_GUARD: LazyLock> = LazyLock::new(|| Mutex::new(())); async fn seed(handle: JobHandle) -> String { match jobs::try_reserve_and_insert(handle, usize::MAX).await { @@ -37,6 +42,7 @@ async fn now_ms_returns_a_recent_unix_ms() { #[tokio::test] async fn insert_then_get_round_trips_the_record() { + let _guard = JOBS_TEST_GUARD.lock().await; let id = "lifecycle-insert-get"; seed(JobHandle { record: rec(id, JobStatus::Running, None), @@ -59,6 +65,7 @@ async fn get_returns_none_for_unknown_id() { #[tokio::test] async fn list_all_includes_inserted_jobs() { + let _guard = JOBS_TEST_GUARD.lock().await; let id1 = "lifecycle-list-all-1"; let id2 = "lifecycle-list-all-2"; seed(JobHandle { @@ -81,6 +88,7 @@ async fn list_all_includes_inserted_jobs() { #[tokio::test] async fn running_count_excludes_terminal_states() { + let _guard = JOBS_TEST_GUARD.lock().await; let running_id = "lifecycle-rc-running"; let finished_id = "lifecycle-rc-finished"; let killed_id = "lifecycle-rc-killed"; @@ -125,6 +133,7 @@ async fn running_count_excludes_terminal_states() { #[tokio::test] async fn remove_old_retention_matrix() { + let _guard = JOBS_TEST_GUARD.lock().await; let stale_id = "lifecycle-remove-old-stale"; let running_id = "lifecycle-remove-old-running"; let fresh_id = "lifecycle-remove-old-fresh"; @@ -171,6 +180,7 @@ async fn remove_old_retention_matrix() { #[tokio::test] async fn concurrent_inserts_and_lookups_dont_deadlock() { + let _guard = JOBS_TEST_GUARD.lock().await; let mut handles = Vec::new(); for i in 0..50 { let h = tokio::spawn(async move {