From aa20eeb6026b52f025d0685edde75e94616f30dc Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Mon, 20 Jul 2026 18:06:18 -0300 Subject: [PATCH 01/11] (MOT-4107) refactor(harness): author integration scenarios as Rust builder modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec revision 2026-07-20 (harness-evaluation): the authored layer is code, not YAML. One builder module per scenario under src/scenarios/ registers in a code registry; the authored structs are shared with the compiler and never serialized, so authored-scenario.v1.json, its golden test, the authored round trip, and the init template CLI are gone. Selection (--scenario id|slug|all), quarantine semantics, validate/render, and all five committed compiled snapshots are unchanged — the snapshots passing unmodified proves the ported modules compile to byte-identical fixtures. - src/scenarios/{builder,mod}.rs: data-only typed builders + registry with compile-and-register-exactly-once tests - five scenario modules ported from scenarios/*/scenario.yaml (306 lines) - fixtures: discovery selects from the registry; loading compiles a registered entry against scenarios/system-prompt.txt - authored-only types drop Serialize/Deserialize/JsonSchema; compiled-shared types (DeadlinesV1, ReleaseV1, FaultKind, invariant parameter structs) keep wire derives - CLI: init removed; validate/render unchanged in behavior - README: builder-module authoring workflow --- harness/evals/integration/README.md | 128 +- .../crash-recovery-507/scenario.yaml | 64 - .../exactly-once-function/scenario.yaml | 49 - .../scenarios/hold-mutation-505/scenario.yaml | 76 - .../hook-held-release-506/scenario.yaml | 95 - .../scenarios/streamed-text/scenario.yaml | 22 - .../schemas/authored-scenario.v1.json | 1740 ----------------- harness/evals/integration/src/expand.rs | 2 +- .../evals/integration/src/expand/render.rs | 9 - .../evals/integration/src/expand/templates.rs | 4 +- harness/evals/integration/src/expand/tests.rs | 7 +- harness/evals/integration/src/fixtures.rs | 11 +- .../integration/src/fixtures/discovery.rs | 179 +- .../evals/integration/src/fixtures/loading.rs | 30 +- .../evals/integration/src/fixtures/tests.rs | 153 +- harness/evals/integration/src/lib.rs | 1 + harness/evals/integration/src/main.rs | 141 +- .../integration/src/scenarios/builder.rs | 433 ++++ .../src/scenarios/crash_recovery_507.rs | 58 + .../src/scenarios/exactly_once_function.rs | 43 + .../src/scenarios/hold_mutation_505.rs | 77 + .../src/scenarios/hook_held_release_506.rs | 95 + .../evals/integration/src/scenarios/mod.rs | 97 + .../src/scenarios/streamed_text.rs | 21 + .../src/types/scenario/authored.rs | 119 +- .../src/types/scenario/expectations.rs | 38 +- .../integration/tests/scenario_compilation.rs | 63 +- harness/evals/integration/tests/schemas.rs | 68 +- 28 files changed, 1121 insertions(+), 2702 deletions(-) delete mode 100644 harness/evals/integration/scenarios/crash-recovery-507/scenario.yaml delete mode 100644 harness/evals/integration/scenarios/exactly-once-function/scenario.yaml delete mode 100644 harness/evals/integration/scenarios/hold-mutation-505/scenario.yaml delete mode 100644 harness/evals/integration/scenarios/hook-held-release-506/scenario.yaml delete mode 100644 harness/evals/integration/scenarios/streamed-text/scenario.yaml delete mode 100644 harness/evals/integration/schemas/authored-scenario.v1.json create mode 100644 harness/evals/integration/src/scenarios/builder.rs create mode 100644 harness/evals/integration/src/scenarios/crash_recovery_507.rs create mode 100644 harness/evals/integration/src/scenarios/exactly_once_function.rs create mode 100644 harness/evals/integration/src/scenarios/hold_mutation_505.rs create mode 100644 harness/evals/integration/src/scenarios/hook_held_release_506.rs create mode 100644 harness/evals/integration/src/scenarios/mod.rs create mode 100644 harness/evals/integration/src/scenarios/streamed_text.rs diff --git a/harness/evals/integration/README.md b/harness/evals/integration/README.md index ec3385b6b..02158ee6c 100644 --- a/harness/evals/integration/README.md +++ b/harness/evals/integration/README.md @@ -43,84 +43,80 @@ byte-stable result contract to be identical. A mismatch is a runner error. ## Create a scenario -Each scenario is one file: `scenarios//scenario.yaml`. Model/provider, -session id, idempotency key, native function policy, run-scoped function ids, -request matchers, response frames, common completion checks, and system -prompt hash are inferred. +Each scenario is one Rust builder module: `src/scenarios/.rs`, a +function that builds the authored data through the typed builders in +`src/scenarios/builder.rs` and registers it in `src/scenarios/mod.rs`. There +is no YAML layer — the authored shape is enforced by the type system at +`cargo build` and is never serialized. Model/provider, session id, +idempotency key, native function policy, run-scoped function ids, request +matchers, response frames, common completion checks, and system prompt hash +are inferred by the compiler. -```bash -harness-integration init \ - --id C-E2E-010 \ - --name my-function-case \ - --description "The allowed function runs once." \ - --kind function +A typical authored function scenario is: -harness-integration validate --scenario my-function-case -harness-integration render C-E2E-010 +```rust +// src/scenarios/my_function_case.rs +pub(super) fn scenario() -> AuthoredScenario { + AuthoredScenario::new("C-E2E-010", "The allowed function runs once.") + .send(Send::message("Call the recorder once.")) + .function( + "record", + Function::new( + "Record one value.", + json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + }), + json!({ + "content": [{ "type": "text", "text": "recorded" }], + "is_error": false + }), + ), + ) + .generation(Reply::function_call("record", json!({ "value": "expected" }))) + .generation(Reply::text("recorded once")) + .expect( + Expect::new() + .assistant_text("recorded once") + .call(TargetCall::counted("record", 1).payload(json!({ "value": "expected" }))), + ) +} ``` -`init` supports `text`, `function`, `hook`, and `crash` templates and refuses -to overwrite an existing directory or reuse an existing scenario id. New -templates are runnable by default; set `quarantine: true` only for a known -reproduction that should be excluded from `run --scenario all`. `render` -prints deterministic canonical JSON with the complete compiled request, -router script, expectations, and system prompt. +Add the module and its slug to the list in `src/scenarios/mod.rs`, then: -A typical authored function scenario is: - -```yaml -schema_version: "1" -id: C-E2E-010 -description: The allowed function runs once. - -send: - message: Call the recorder once. - -functions: - record: - description: Record one value. - request_schema: - type: object - additionalProperties: false - properties: - value: { type: string } - required: [value] - response: - content: - - { type: text, text: recorded } - is_error: false - -router: - generations: - - reply: - type: function_call - function: record - arguments: { value: expected } - - reply: - type: text - text: recorded once - -expect: - assistant_text: recorded once - calls: - - function: record - count: 1 - payload: { value: expected } +```bash +cargo test # builder, snapshot, and contract tests +REGEN_SCENARIO_SNAPSHOTS=1 cargo test --test scenario_compilation +harness-integration validate --scenario all +harness-integration render C-E2E-010 ``` -Function aliases become `::`. Set `expose: false` for -hook-only functions. `send.allow` can narrow the exposed aliases or be an -empty list to disable dispatch. Typed text and function-call replies cover -normal cases; `match_overrides` and `type: raw` remain escape hatches for -recovery boundaries and unusual wire contracts. +Builders produce data only — a builder that derives scenario content from +control flow is rejected in review. The compiled snapshot under +`tests/snapshots/.compiled.json` is the review artifact; commit the +regenerated snapshot with the new module. New scenarios are runnable by +default; chain `.quarantine()` only for a known reproduction that should be +excluded from `run --scenario all`. `render` prints deterministic canonical +JSON with the complete compiled request, router script, expectations, and +system prompt. + +Function aliases become `::`. Chain `.hidden()` for hook-only +functions. `Send::message(...).allow([...])` can narrow the exposed aliases +or be an empty list to disable dispatch. Typed text and function-call replies +cover normal cases; `.match_overrides(...)` and `RouterReplyV1::Raw` remain +escape hatches for recovery boundaries and unusual wire contracts. Timeout defaults are 60 seconds for readiness, 60 seconds for the scenario, -and 15 seconds for teardown. Positive values can be overridden under -`timeouts`; one readiness budget is shared by the full probe/arm sequence. +and 15 seconds for teardown. Positive values can be overridden with the +`*_timeout_ms` builders; one readiness budget is shared by the full probe/arm +sequence. ## Checked-in scenarios -| id | directory | status | +| id | slug | status | |---|---|---| | C-E2E-001 | `streamed-text` | streamed text reaches durable completion | | C-E2E-002 | `exactly-once-function` | a native function executes exactly once | diff --git a/harness/evals/integration/scenarios/crash-recovery-507/scenario.yaml b/harness/evals/integration/scenarios/crash-recovery-507/scenario.yaml deleted file mode 100644 index 848d9c71a..000000000 --- a/harness/evals/integration/scenarios/crash-recovery-507/scenario.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# C-E2E-507 — crash recovery closes the interrupted function call. -# Reproduction of https://github.com/iii-hq/workers/issues/507. -schema_version: "1" -id: C-E2E-507 -description: An engine crash during a dispatched function call must not leave the call dangling or the session unusable. -quarantine: true - -send: - message: Call the recorder once. - -functions: - record: - description: Record one integration fixture value. - request_schema: - type: object - additionalProperties: false - properties: - value: { type: string } - required: [value] - response: - content: - - { type: text, text: recorded } - is_error: false - -router: - generations: - - reply: - type: function_call - function: record - arguments: { value: expected } - usage: { input: 8, output: 4 } - - reply: - type: text - text: recovered - usage: { input: 20, output: 2 } - # Recovery can legitimately reconstruct the second request differently; - # this reproduction grades the durable outcome instead. - match_overrides: - request_id: - mode: regex - pattern: "^t_[0-9a-f]{32}:[0-9]+$" - system_prompt: { mode: present } - messages: { mode: present } - tools: { mode: present } - -fault: - kind: engine_sigkill - -timeouts: - scenario_ms: 120000 - -expect: - message_counts: - user: 1 - assistant: 2 - function_result: 1 - assistant_text: recovered - calls_closed: true - function_results: - - function_call_id: call-1 - calls: - - function: record - count: 1 - payload: { value: expected } diff --git a/harness/evals/integration/scenarios/exactly-once-function/scenario.yaml b/harness/evals/integration/scenarios/exactly-once-function/scenario.yaml deleted file mode 100644 index e44cd0c01..000000000 --- a/harness/evals/integration/scenarios/exactly-once-function/scenario.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# C-E2E-002 — an allow-listed function executes exactly once. -schema_version: "1" -id: C-E2E-002 -description: An allow-listed native function executes exactly once with a durable result. - -send: - message: Call the recorder once. - -functions: - record: - description: Record one integration fixture value. - request_schema: - type: object - additionalProperties: false - properties: - value: { type: string } - required: [value] - response: - content: - - { type: text, text: recorded } - is_error: false - -router: - generations: - - reply: - type: function_call - function: record - arguments: { value: expected } - usage: { input: 8, output: 4 } - - reply: - type: text - text: recorded once - usage: { input: 18, output: 2 } - -expect: - message_counts: - user: 1 - assistant: 2 - function_result: 1 - assistant_text: recorded once - function_results: - - function: record - content: - - { type: text, text: recorded } - is_error: false - calls: - - function: record - count: 1 - payload: { value: expected } diff --git a/harness/evals/integration/scenarios/hold-mutation-505/scenario.yaml b/harness/evals/integration/scenarios/hold-mutation-505/scenario.yaml deleted file mode 100644 index 39ebc0f61..000000000 --- a/harness/evals/integration/scenarios/hold-mutation-505/scenario.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# C-E2E-505 — a holding hook's mutation reaches the released call. -# Reproduction of https://github.com/iii-hq/workers/issues/505. -schema_version: "1" -id: C-E2E-505 -description: A pre-trigger hook that holds and mutates must apply its mutation to the released call. -quarantine: true - -send: - message: Call the recorder once. - -functions: - record: - description: Record one integration fixture value. - request_schema: - type: object - additionalProperties: false - properties: - value: { type: string } - required: [value] - response: - content: - - { type: text, text: recorded } - is_error: false - hook-gate: - description: Hold the call and stamp approval context onto its arguments. - request_schema: - type: object - response: - decision: hold - mutations: - arguments: - value: expected+approved - expose: false - -bindings: - - trigger: hook_pre_trigger - function: hook-gate - functions: [record] - priority: 10 - -release: - action: execute - -router: - generations: - - reply: - type: function_call - function: record - arguments: { value: expected } - usage: { input: 8, output: 4 } - - reply: - type: text - text: approved and recorded - usage: { input: 20, output: 3 } - match_overrides: - request_id: - mode: regex - pattern: "^t_[0-9a-f]{32}:[0-9]+$" - system_prompt: { mode: present } - messages: { mode: present } - tools: { mode: present } - -expect: - calls_closed: true - calls: - - function: record - count: 1 - payload: { value: expected+approved } - - function: hook-gate - count: 1 - payload_subset: - point: pre_trigger - call: - id: call-1 - function_id: "{{run_id}}::record" - arguments: { value: expected } diff --git a/harness/evals/integration/scenarios/hook-held-release-506/scenario.yaml b/harness/evals/integration/scenarios/hook-held-release-506/scenario.yaml deleted file mode 100644 index bf5c98cee..000000000 --- a/harness/evals/integration/scenarios/hook-held-release-506/scenario.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# C-E2E-506 — released held calls retain hook-mutated arguments. -# Reproduction of https://github.com/iii-hq/workers/issues/506. -schema_version: "1" -id: C-E2E-506 -description: A held call released for execution must run with the arguments produced by earlier hooks. -quarantine: true - -send: - message: Call the recorder once. - -functions: - record: - description: Record one integration fixture value. - request_schema: - type: object - additionalProperties: false - properties: - value: { type: string } - required: [value] - response: - content: - - { type: text, text: recorded } - is_error: false - hook-mutate: - description: Inject validated scope into the arguments. - request_schema: - type: object - response: - decision: continue - mutations: - arguments: - value: expected+scope - expose: false - hook-hold: - description: Hold every consulted call for explicit approval. - request_schema: - type: object - response: - decision: hold - expose: false - -bindings: - - trigger: hook_pre_trigger - function: hook-mutate - functions: [record] - priority: 10 - - trigger: hook_pre_trigger - function: hook-hold - functions: [record] - priority: 20 - -release: - action: execute - -router: - generations: - - reply: - type: function_call - function: record - arguments: { value: expected } - usage: { input: 8, output: 4 } - - reply: - type: text - text: released and recorded - usage: { input: 20, output: 3 } - match_overrides: - request_id: - mode: regex - pattern: "^t_[0-9a-f]{32}:[0-9]+$" - system_prompt: { mode: present } - messages: { mode: present } - tools: { mode: present } - -expect: - calls_closed: true - calls: - - function: record - count: 1 - payload: { value: expected+scope } - - function: hook-mutate - count: 1 - payload_subset: - point: pre_trigger - call: - id: call-1 - function_id: "{{run_id}}::record" - arguments: { value: expected } - - function: hook-hold - count: 1 - payload_subset: - point: pre_trigger - call: - id: call-1 - function_id: "{{run_id}}::record" - arguments: { value: expected+scope } diff --git a/harness/evals/integration/scenarios/streamed-text/scenario.yaml b/harness/evals/integration/scenarios/streamed-text/scenario.yaml deleted file mode 100644 index b8eb0e26e..000000000 --- a/harness/evals/integration/scenarios/streamed-text/scenario.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# C-E2E-001 — streamed text reaches durable completion. -schema_version: "1" -id: C-E2E-001 -description: Streamed text reaches durable completion through the real queue and turn loop. - -send: - message: Return the fixture phrase. - -router: - generations: - - reply: - type: text - text: fixture complete - chunks: ["fixture ", "complete"] - usage: { input: 8, output: 2 } - -expect: - message_counts: - user: 1 - assistant: 1 - function_result: 0 - assistant_text: fixture complete diff --git a/harness/evals/integration/schemas/authored-scenario.v1.json b/harness/evals/integration/schemas/authored-scenario.v1.json deleted file mode 100644 index 55a2c9065..000000000 --- a/harness/evals/integration/schemas/authored-scenario.v1.json +++ /dev/null @@ -1,1740 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "definitions": { - "AssistantMessage": { - "additionalProperties": false, - "properties": { - "content": { - "items": { - "$ref": "#/definitions/ContentBlock" - }, - "type": "array" - }, - "error_kind": { - "anyOf": [ - { - "$ref": "#/definitions/ErrorKind" - }, - { - "type": "null" - } - ] - }, - "error_message": { - "type": [ - "string", - "null" - ] - }, - "model": { - "type": "string" - }, - "native_stop_reason": { - "type": [ - "string", - "null" - ] - }, - "provider": { - "type": "string" - }, - "role": { - "$ref": "#/definitions/AssistantRoleTag" - }, - "stop_reason": { - "$ref": "#/definitions/StopReason" - }, - "timestamp": { - "format": "int64", - "type": "integer" - }, - "usage": { - "anyOf": [ - { - "$ref": "#/definitions/Usage" - }, - { - "type": "null" - } - ] - }, - "warnings": { - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - } - }, - "required": [ - "content", - "model", - "provider", - "role", - "stop_reason", - "timestamp" - ], - "type": "object" - }, - "AssistantMessageEvent": { - "description": "The frozen 15-variant streaming vocabulary (`llm-router/src/types/events.rs:53`).\n\nDelta variants carry `partial` as an Option, mirroring the router: the current wire format is the slim delta (`partial` omitted, readers accumulate); a legacy fat delta (`partial: Some`) is an authoritative snapshot from old producers. Fixtures may author either form.", - "oneOf": [ - { - "additionalProperties": false, - "properties": { - "partial": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "start" - ], - "type": "string" - } - }, - "required": [ - "partial", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "partial": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "text_start" - ], - "type": "string" - } - }, - "required": [ - "partial", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "delta": { - "type": "string" - }, - "partial": { - "anyOf": [ - { - "$ref": "#/definitions/AssistantMessage" - }, - { - "type": "null" - } - ] - }, - "type": { - "enum": [ - "text_delta" - ], - "type": "string" - } - }, - "required": [ - "delta", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "partial": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "text_end" - ], - "type": "string" - } - }, - "required": [ - "partial", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "partial": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "thinking_start" - ], - "type": "string" - } - }, - "required": [ - "partial", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "delta": { - "type": "string" - }, - "partial": { - "anyOf": [ - { - "$ref": "#/definitions/AssistantMessage" - }, - { - "type": "null" - } - ] - }, - "type": { - "enum": [ - "thinking_delta" - ], - "type": "string" - } - }, - "required": [ - "delta", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "partial": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "thinking_end" - ], - "type": "string" - } - }, - "required": [ - "partial", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "partial": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "functioncall_start" - ], - "type": "string" - } - }, - "required": [ - "partial", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "delta": { - "type": "string" - }, - "id": { - "type": "string" - }, - "partial": { - "anyOf": [ - { - "$ref": "#/definitions/AssistantMessage" - }, - { - "type": "null" - } - ] - }, - "type": { - "enum": [ - "functioncall_delta" - ], - "type": "string" - } - }, - "required": [ - "delta", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "partial": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "functioncall_end" - ], - "type": "string" - } - }, - "required": [ - "partial", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "type": { - "enum": [ - "usage" - ], - "type": "string" - }, - "usage": { - "$ref": "#/definitions/Usage" - } - }, - "required": [ - "type", - "usage" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "type": { - "enum": [ - "ping" - ], - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "error_kind": { - "anyOf": [ - { - "$ref": "#/definitions/ErrorKind" - }, - { - "type": "null" - } - ] - }, - "error_message": { - "type": [ - "string", - "null" - ] - }, - "stop_reason": { - "$ref": "#/definitions/StopReason" - }, - "type": { - "enum": [ - "stop" - ], - "type": "string" - } - }, - "required": [ - "stop_reason", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "message": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "done" - ], - "type": "string" - } - }, - "required": [ - "message", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "error": { - "$ref": "#/definitions/AssistantMessage" - }, - "type": { - "enum": [ - "error" - ], - "type": "string" - } - }, - "required": [ - "error", - "type" - ], - "type": "object" - } - ] - }, - "AssistantRoleTag": { - "enum": [ - "assistant" - ], - "type": "string" - }, - "ContentBlock": { - "oneOf": [ - { - "additionalProperties": false, - "properties": { - "text": { - "type": "string" - }, - "type": { - "enum": [ - "text" - ], - "type": "string" - } - }, - "required": [ - "text", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "data": { - "type": "string" - }, - "mime": { - "type": "string" - }, - "type": { - "enum": [ - "image" - ], - "type": "string" - } - }, - "required": [ - "data", - "mime", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "signature": { - "type": [ - "string", - "null" - ] - }, - "text": { - "type": "string" - }, - "type": { - "enum": [ - "thinking" - ], - "type": "string" - } - }, - "required": [ - "text", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "data": { - "type": "string" - }, - "type": { - "enum": [ - "redacted_thinking" - ], - "type": "string" - } - }, - "required": [ - "data", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "arguments": true, - "function_id": { - "type": "string" - }, - "id": { - "type": "string" - }, - "type": { - "enum": [ - "function_call" - ], - "type": "string" - } - }, - "required": [ - "arguments", - "function_id", - "id", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "content": { - "items": { - "$ref": "#/definitions/ContentBlock" - }, - "type": "array" - }, - "function_call_id": { - "type": "string" - }, - "is_error": { - "type": [ - "boolean", - "null" - ] - }, - "type": { - "enum": [ - "function_result" - ], - "type": "string" - } - }, - "required": [ - "content", - "function_call_id", - "type" - ], - "type": "object" - } - ] - }, - "DeadlinesV1": { - "additionalProperties": false, - "properties": { - "readiness_ms": { - "default": 60000, - "format": "uint64", - "minimum": 1.0, - "type": "integer" - }, - "scenario_ms": { - "default": 60000, - "format": "uint64", - "minimum": 1.0, - "type": "integer" - }, - "teardown_ms": { - "default": 15000, - "format": "uint64", - "minimum": 1.0, - "type": "integer" - } - }, - "type": "object" - }, - "ErrorKind": { - "enum": [ - "auth_expired", - "rate_limited", - "context_overflow", - "transient", - "permanent" - ], - "type": "string" - }, - "ErrorShape": { - "additionalProperties": false, - "properties": { - "code": { - "type": "string" - }, - "message": { - "type": "string" - } - }, - "required": [ - "code", - "message" - ], - "type": "object" - }, - "ExpectationsV1": { - "additionalProperties": false, - "description": "Typed oracle vocabulary. Common send/completion/lifecycle/router checks have defaults, leaving each fixture to state only scenario-specific facts.", - "properties": { - "assistant_text": { - "type": [ - "string", - "null" - ] - }, - "calls": { - "items": { - "$ref": "#/definitions/TargetCallsExpectationV1" - }, - "type": "array" - }, - "calls_closed": { - "type": "boolean" - }, - "function_results": { - "items": { - "$ref": "#/definitions/FunctionResultExpectationV1" - }, - "type": "array" - }, - "lifecycle": { - "$ref": "#/definitions/LifecycleExpectationV1" - }, - "message_counts": { - "anyOf": [ - { - "$ref": "#/definitions/MessageCountsExpectationV1" - }, - { - "type": "null" - } - ] - }, - "no_duplicates": { - "type": "boolean" - }, - "send_flags": { - "$ref": "#/definitions/SendFlagsExpectationV1" - }, - "terminal": { - "$ref": "#/definitions/TerminalExpectationV1" - } - }, - "type": "object" - }, - "FaultKind": { - "enum": [ - "engine_sigkill" - ], - "type": "string" - }, - "FaultV1": { - "additionalProperties": false, - "properties": { - "after_target_calls": { - "format": "uint64", - "minimum": 1.0, - "type": "integer" - }, - "function": { - "description": "Controlled function alias to interrupt. Omitted means the first authored function call.", - "type": [ - "string", - "null" - ] - }, - "kind": { - "$ref": "#/definitions/FaultKind" - }, - "restart_delay_ms": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - "FunctionResultExpectationV1": { - "additionalProperties": false, - "properties": { - "content": { - "items": true, - "type": [ - "array", - "null" - ] - }, - "function": { - "description": "Optional function alias; omitted when any result closing the call is acceptable (for example, a synthesized crash-recovery error).", - "type": [ - "string", - "null" - ] - }, - "function_call_id": { - "default": "call-1", - "type": "string" - }, - "is_error": { - "type": [ - "boolean", - "null" - ] - } - }, - "type": "object" - }, - "GenerationMatchOverridesV1": { - "additionalProperties": false, - "properties": { - "max_output_tokens": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "messages": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "metadata": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "model": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "provider": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "provider_options": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "request_id": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "response_format": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "system_prompt": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "thinking_level": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "tools": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - }, - "writer_ref": { - "anyOf": [ - { - "$ref": "#/definitions/JsonMatcherV1" - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "JsonMatcherV1": { - "oneOf": [ - { - "additionalProperties": false, - "properties": { - "mode": { - "enum": [ - "absent" - ], - "type": "string" - } - }, - "required": [ - "mode" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "mode": { - "enum": [ - "present" - ], - "type": "string" - } - }, - "required": [ - "mode" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "mode": { - "enum": [ - "regex" - ], - "type": "string" - }, - "pattern": { - "type": "string" - } - }, - "required": [ - "mode", - "pattern" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "expected": { - "type": "string" - }, - "mode": { - "enum": [ - "sha256" - ], - "type": "string" - } - }, - "required": [ - "expected", - "mode" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "expected": true, - "mode": { - "enum": [ - "exact" - ], - "type": "string" - }, - "normalize": { - "items": { - "$ref": "#/definitions/JsonNormalizerV1" - }, - "type": [ - "array", - "null" - ] - } - }, - "required": [ - "expected", - "mode" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "expected": true, - "mode": { - "enum": [ - "subset" - ], - "type": "string" - }, - "normalize": { - "items": { - "$ref": "#/definitions/JsonNormalizerV1" - }, - "type": [ - "array", - "null" - ] - } - }, - "required": [ - "expected", - "mode" - ], - "type": "object" - } - ] - }, - "JsonNormalizerV1": { - "additionalProperties": false, - "properties": { - "operation": { - "$ref": "#/definitions/NormalizerOperation" - }, - "pointer": { - "description": "RFC 6901 JSON Pointer.", - "type": "string" - }, - "replacement": { - "description": "Required for `replace`; forbidden for `delete`." - } - }, - "required": [ - "operation", - "pointer" - ], - "type": "object" - }, - "LifecycleExpectationV1": { - "additionalProperties": false, - "properties": { - "allow_identical_duplicates": { - "default": true, - "type": "boolean" - } - }, - "type": "object" - }, - "MessageCountsExpectationV1": { - "additionalProperties": false, - "properties": { - "assistant": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "function_result": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "user": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - } - }, - "required": [ - "assistant", - "function_result", - "user" - ], - "type": "object" - }, - "ModelFixtureV1": { - "additionalProperties": false, - "description": "Mirror of the catalog `Model` (`llm-router/src/types/model.rs:43`).", - "properties": { - "context_window": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "display_name": { - "type": [ - "string", - "null" - ] - }, - "id": { - "type": "string" - }, - "input_limit": { - "format": "uint64", - "minimum": 0.0, - "type": [ - "integer", - "null" - ] - }, - "max_output_tokens": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "pricing": { - "anyOf": [ - { - "$ref": "#/definitions/PricingV1" - }, - { - "type": "null" - } - ] - }, - "provider": { - "type": "string" - }, - "reasoning_efforts": { - "items": { - "$ref": "#/definitions/ReasoningEffortV1" - }, - "type": [ - "array", - "null" - ] - }, - "supports_cache": { - "type": [ - "boolean", - "null" - ] - }, - "supports_structured_output": { - "type": [ - "boolean", - "null" - ] - }, - "supports_thinking": { - "type": [ - "boolean", - "null" - ] - }, - "supports_tools": { - "type": [ - "boolean", - "null" - ] - }, - "supports_vision": { - "type": [ - "boolean", - "null" - ] - }, - "supports_xhigh": { - "type": [ - "boolean", - "null" - ] - }, - "thinking_budgets": { - "additionalProperties": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "type": [ - "object", - "null" - ] - } - }, - "required": [ - "context_window", - "id", - "max_output_tokens", - "provider" - ], - "type": "object" - }, - "NormalizerOperation": { - "enum": [ - "delete", - "replace" - ], - "type": "string" - }, - "PricingV1": { - "additionalProperties": false, - "properties": { - "cache_read": { - "format": "double", - "type": [ - "number", - "null" - ] - }, - "cache_write": { - "format": "double", - "type": [ - "number", - "null" - ] - }, - "input": { - "format": "double", - "type": [ - "number", - "null" - ] - }, - "output": { - "format": "double", - "type": [ - "number", - "null" - ] - } - }, - "type": "object" - }, - "ReasoningEffortV1": { - "additionalProperties": false, - "properties": { - "description": { - "type": [ - "string", - "null" - ] - }, - "effort": { - "type": "string" - } - }, - "required": [ - "effort" - ], - "type": "object" - }, - "ReleaseActionV1": { - "enum": [ - "execute", - "deliver" - ], - "type": "string" - }, - "ReleaseV1": { - "additionalProperties": false, - "properties": { - "action": { - "$ref": "#/definitions/ReleaseActionV1" - }, - "function_call_id": { - "type": "string" - } - }, - "required": [ - "action" - ], - "type": "object" - }, - "RouterChatResponse": { - "additionalProperties": false, - "description": "Mirror of `ChatResponse` (`llm-router/src/types/router.rs:54`).", - "properties": { - "error": { - "anyOf": [ - { - "$ref": "#/definitions/ErrorShape" - }, - { - "type": "null" - } - ] - }, - "model": { - "type": "string" - }, - "ok": { - "type": "boolean" - }, - "provider": { - "type": "string" - }, - "stop_reason": { - "anyOf": [ - { - "$ref": "#/definitions/StopReason" - }, - { - "type": "null" - } - ] - }, - "usage": { - "anyOf": [ - { - "$ref": "#/definitions/Usage" - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "model", - "ok", - "provider" - ], - "type": "object" - }, - "RouterReplyV1": { - "oneOf": [ - { - "additionalProperties": false, - "properties": { - "chunks": { - "description": "Non-empty chunks produce the complete streaming frame sequence. Omitted chunks produce one terminal `done` frame.", - "items": { - "type": "string" - }, - "type": "array" - }, - "text": { - "type": "string" - }, - "type": { - "enum": [ - "text" - ], - "type": "string" - }, - "usage": { - "anyOf": [ - { - "$ref": "#/definitions/Usage" - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "text", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "arguments": true, - "function": { - "description": "Function alias from `functions`.", - "type": "string" - }, - "id": { - "description": "Defaults to `call-`.", - "type": [ - "string", - "null" - ] - }, - "type": { - "enum": [ - "function_call" - ], - "type": "string" - }, - "usage": { - "anyOf": [ - { - "$ref": "#/definitions/Usage" - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "arguments", - "function", - "type" - ], - "type": "object" - }, - { - "additionalProperties": false, - "description": "Full strict wire contract for cases the typed replies cannot express.", - "properties": { - "frames": { - "items": { - "$ref": "#/definitions/AssistantMessageEvent" - }, - "type": "array" - }, - "response": { - "$ref": "#/definitions/RouterChatResponse" - }, - "type": { - "enum": [ - "raw" - ], - "type": "string" - } - }, - "required": [ - "frames", - "response", - "type" - ], - "type": "object" - } - ] - }, - "ScenarioFunctionV1": { - "additionalProperties": false, - "properties": { - "description": { - "type": "string" - }, - "expose": { - "description": "Exposed to the model by default. Hook-only controlled functions set this to false.", - "type": "boolean" - }, - "request_schema": { - "additionalProperties": true, - "type": "object" - }, - "response": true - }, - "required": [ - "description", - "request_schema", - "response" - ], - "type": "object" - }, - "ScenarioGenerationV1": { - "additionalProperties": false, - "properties": { - "match_overrides": { - "allOf": [ - { - "$ref": "#/definitions/GenerationMatchOverridesV1" - } - ], - "description": "Escape hatch for fields whose history is intentionally unstable, such as the post-crash request in a recovery reproduction." - }, - "reply": { - "$ref": "#/definitions/RouterReplyV1" - } - }, - "required": [ - "reply" - ], - "type": "object" - }, - "ScenarioRouterV1": { - "additionalProperties": false, - "properties": { - "generations": { - "items": { - "$ref": "#/definitions/ScenarioGenerationV1" - }, - "type": "array" - }, - "model": { - "anyOf": [ - { - "$ref": "#/definitions/ModelFixtureV1" - }, - { - "type": "null" - } - ], - "description": "Omitted for the deterministic `fixture-model` / `scripted` catalog entry used by the integration stack." - } - }, - "required": [ - "generations" - ], - "type": "object" - }, - "ScenarioSendV1": { - "additionalProperties": false, - "properties": { - "allow": { - "description": "Allowed function aliases. Omitted means every function whose `expose` flag is true; an empty list disables function dispatch.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - }, - "idempotency_key": { - "description": "Omitted values are derived deterministically from the scenario id.", - "type": [ - "string", - "null" - ] - }, - "message": { - "type": "string" - } - }, - "required": [ - "message" - ], - "type": "object" - }, - "SchemaVersion1": { - "description": "The literal string `\"1\"`; any other value is a schema error.", - "enum": [ - "1" - ], - "type": "string" - }, - "SendFlagsExpectationV1": { - "additionalProperties": false, - "properties": { - "deduplicated": { - "default": false, - "type": "boolean" - }, - "merged": { - "default": false, - "type": "boolean" - }, - "queued": { - "default": false, - "type": "boolean" - } - }, - "type": "object" - }, - "StopReason": { - "enum": [ - "end", - "length", - "function_call", - "aborted", - "error" - ], - "type": "string" - }, - "TargetCallsExpectationV1": { - "additionalProperties": false, - "properties": { - "count": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "function": { - "type": "string" - }, - "payload": true, - "payload_subset": true - }, - "required": [ - "count", - "function" - ], - "type": "object" - }, - "TerminalExpectationV1": { - "additionalProperties": false, - "properties": { - "pending_calls": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "queued_messages": { - "format": "uint64", - "minimum": 0.0, - "type": "integer" - }, - "status": { - "$ref": "#/definitions/TerminalStatusV1" - } - }, - "required": [ - "pending_calls", - "queued_messages", - "status" - ], - "type": "object" - }, - "TerminalStatusV1": { - "enum": [ - "completed", - "failed", - "cancelled" - ], - "type": "string" - }, - "TriggerBindingSpecV1": { - "additionalProperties": false, - "properties": { - "function": { - "description": "Controlled function alias invoked by the trigger.", - "type": "string" - }, - "functions": { - "description": "Exposed function aliases selected by this hook.", - "items": { - "type": "string" - }, - "type": "array" - }, - "priority": { - "format": "int64", - "type": "integer" - }, - "trigger": { - "$ref": "#/definitions/TriggerKindV1" - } - }, - "required": [ - "function", - "functions", - "priority", - "trigger" - ], - "type": "object" - }, - "TriggerKindV1": { - "enum": [ - "hook_pre_trigger" - ], - "type": "string" - }, - "Usage": { - "additionalProperties": false, - "properties": { - "cache_read": { - "format": "uint64", - "minimum": 0.0, - "type": [ - "integer", - "null" - ] - }, - "cache_write": { - "format": "uint64", - "minimum": 0.0, - "type": [ - "integer", - "null" - ] - }, - "cost_usd": { - "format": "double", - "type": [ - "number", - "null" - ] - }, - "input": { - "format": "uint64", - "minimum": 0.0, - "type": [ - "integer", - "null" - ] - }, - "output": { - "format": "uint64", - "minimum": 0.0, - "type": [ - "integer", - "null" - ] - }, - "reasoning": { - "format": "uint64", - "minimum": 0.0, - "type": [ - "integer", - "null" - ] - } - }, - "type": "object" - } - }, - "description": "The single-file scenario authors maintain in `scenarios//scenario.yaml`.", - "properties": { - "bindings": { - "items": { - "$ref": "#/definitions/TriggerBindingSpecV1" - }, - "type": "array" - }, - "description": { - "type": "string" - }, - "expect": { - "$ref": "#/definitions/ExpectationsV1" - }, - "fault": { - "anyOf": [ - { - "$ref": "#/definitions/FaultV1" - }, - { - "type": "null" - } - ] - }, - "functions": { - "additionalProperties": { - "$ref": "#/definitions/ScenarioFunctionV1" - }, - "description": "Alias → controlled function. Aliases are expanded to `{{run_id}}::` by the compiler.", - "propertyNames": { - "minLength": 1, - "pattern": "^[A-Za-z0-9_-]+$", - "type": "string" - }, - "type": "object" - }, - "id": { - "maxLength": 128, - "minLength": 1, - "pattern": "^[A-Za-z0-9_-]+$", - "type": "string" - }, - "quarantine": { - "type": "boolean" - }, - "release": { - "anyOf": [ - { - "$ref": "#/definitions/ReleaseV1" - }, - { - "type": "null" - } - ] - }, - "router": { - "$ref": "#/definitions/ScenarioRouterV1" - }, - "schema_version": { - "$ref": "#/definitions/SchemaVersion1" - }, - "send": { - "$ref": "#/definitions/ScenarioSendV1" - }, - "timeouts": { - "$ref": "#/definitions/DeadlinesV1" - } - }, - "required": [ - "description", - "id", - "router", - "schema_version", - "send" - ], - "title": "AuthoredScenarioV1", - "type": "object" -} diff --git a/harness/evals/integration/src/expand.rs b/harness/evals/integration/src/expand.rs index 8ab68f285..dc4999682 100644 --- a/harness/evals/integration/src/expand.rs +++ b/harness/evals/integration/src/expand.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use crate::types::scenario::{AuthoredScenarioV1, CompiledScenarioV1}; use crate::types::script::{ModelFixtureV1, RouterScriptV1}; -pub use render::{render_authored_yaml, render_compiled}; +pub use render::render_compiled; pub use templates::{scenario_template, ScenarioTemplateKind}; pub use tokens::Placeholders; diff --git a/harness/evals/integration/src/expand/render.rs b/harness/evals/integration/src/expand/render.rs index 05e0bc4a0..c9cde2ac6 100644 --- a/harness/evals/integration/src/expand/render.rs +++ b/harness/evals/integration/src/expand/render.rs @@ -2,7 +2,6 @@ use anyhow::Context; use serde_json::json; use super::{expand_compiled_fixture, CompiledFixtureV1}; -use crate::types::scenario::AuthoredScenarioV1; pub(super) fn validate_placeholders(fixture: &CompiledFixtureV1) -> anyhow::Result<()> { render_compiled(fixture) @@ -25,11 +24,3 @@ pub fn render_compiled(fixture: &CompiledFixtureV1) -> anyhow::Result { }))) } -/// Serialize an authored scenario in stable YAML for `init`. -pub fn render_authored_yaml(scenario: &AuthoredScenarioV1) -> anyhow::Result { - let mut rendered = serde_yaml::to_string(scenario)?; - if !rendered.ends_with('\n') { - rendered.push('\n'); - } - Ok(rendered) -} diff --git a/harness/evals/integration/src/expand/templates.rs b/harness/evals/integration/src/expand/templates.rs index b39bafa69..55cfc3a9d 100644 --- a/harness/evals/integration/src/expand/templates.rs +++ b/harness/evals/integration/src/expand/templates.rs @@ -16,7 +16,9 @@ pub enum ScenarioTemplateKind { Crash, } -/// Minimal valid authored scenario used by the non-interactive `init` CLI. +/// Minimal valid authored scenario used as a compact factory by unit and +/// contract tests. New checked-in scenarios are builder modules under +/// `src/scenarios`, not templates. pub fn scenario_template( id: &str, description: &str, diff --git a/harness/evals/integration/src/expand/tests.rs b/harness/evals/integration/src/expand/tests.rs index ae1c3fbbe..de0929153 100644 --- a/harness/evals/integration/src/expand/tests.rs +++ b/harness/evals/integration/src/expand/tests.rs @@ -8,8 +8,7 @@ use crate::types::scenario::{ use crate::types::script::{JsonMatcherV1, SchemaVersion1}; use super::{ - compile_scenario, render_authored_yaml, render_compiled, scenario_template, Placeholders, - ScenarioTemplateKind, + compile_scenario, render_compiled, scenario_template, Placeholders, ScenarioTemplateKind, }; fn minimal(reply: RouterReplyV1) -> AuthoredScenarioV1 { @@ -293,9 +292,7 @@ fn templates_compile_and_render_deterministically() { ScenarioTemplateKind::Crash, ] { let authored = scenario_template("C-E2E-NEW", "A generated scenario.", kind); - let yaml = render_authored_yaml(&authored).unwrap(); - let reparsed: AuthoredScenarioV1 = serde_yaml::from_str(&yaml).unwrap(); - let fixture = compile_scenario(&reparsed, "base\n").unwrap(); + let fixture = compile_scenario(&authored, "base\n").unwrap(); assert_eq!( render_compiled(&fixture).unwrap(), render_compiled(&fixture).unwrap() diff --git a/harness/evals/integration/src/fixtures.rs b/harness/evals/integration/src/fixtures.rs index a02c5bba6..d614703bf 100644 --- a/harness/evals/integration/src/fixtures.rs +++ b/harness/evals/integration/src/fixtures.rs @@ -1,15 +1,16 @@ -//! Scenario loading, validation, discovery, and selection. +//! Scenario compilation, validation, and selection over the code registry +//! in [`crate::scenarios`]. //! -//! Every authoring defect is reported before the stack starts. The public -//! facade preserves the original `fixtures` API while keeping each -//! responsibility in a focused module. +//! Every authoring defect the type system cannot reject is reported before +//! the stack starts. The public facade preserves the original `fixtures` API +//! while keeping each responsibility in a focused module. mod discovery; mod loading; mod script_validation; mod stream_validation; -pub use discovery::{ensure_scenario_id_available, scenario_fixtures}; +pub use discovery::scenario_fixtures; pub use loading::ScenarioFixture; pub use script_validation::validate_script; diff --git a/harness/evals/integration/src/fixtures/discovery.rs b/harness/evals/integration/src/fixtures/discovery.rs index 29256ae67..f599361d7 100644 --- a/harness/evals/integration/src/fixtures/discovery.rs +++ b/harness/evals/integration/src/fixtures/discovery.rs @@ -1,10 +1,10 @@ -use std::path::{Path, PathBuf}; - -use anyhow::Context; +use std::path::Path; use super::loading::ScenarioFixture; +use crate::scenarios::RegisteredScenario; -/// Resolve `--scenario ` and return already-loaded fixtures. +/// Resolve `--scenario ` against the code registry and return +/// already-compiled fixtures. /// /// `include_quarantined` is intended for validation. Explicit id/slug /// selection always includes the requested fixture; for `all`, normal runs @@ -14,22 +14,29 @@ pub fn scenario_fixtures( selector: &str, include_quarantined: bool, ) -> anyhow::Result> { - let dirs = scenario_directories(scenarios_root)?; + select_fixtures( + crate::scenarios::all(), + scenarios_root, + selector, + include_quarantined, + ) +} + +/// Registry-parameterized core so selection semantics stay testable without +/// the checked-in scenario set. +pub(crate) fn select_fixtures( + registered: Vec, + scenarios_root: &Path, + selector: &str, + include_quarantined: bool, +) -> anyhow::Result> { + reject_duplicate_identities(®istered)?; + if selector == "all" { let mut selected = Vec::new(); - let mut ids = std::collections::BTreeMap::new(); - for dir in dirs { - let fixture = ScenarioFixture::load(&dir)?; - if let Some(previous) = ids.insert(fixture.scenario.id.clone(), dir.clone()) { - anyhow::bail!( - "duplicate scenario id {:?} in {} and {}", - fixture.scenario.id, - previous.display(), - dir.display() - ); - } - if include_quarantined || !fixture.scenario.quarantine { - selected.push(fixture); + for entry in ®istered { + if include_quarantined || !entry.authored.quarantine { + selected.push(ScenarioFixture::from_registered(entry, scenarios_root)?); } } if selected.is_empty() { @@ -38,121 +45,55 @@ pub fn scenario_fixtures( } else { " non-quarantined" }; - anyhow::bail!( - "selector \"all\" matched no{qualifier} scenario under {}", - scenarios_root.display() - ); + anyhow::bail!("selector \"all\" matched no{qualifier} registered scenario"); } return Ok(selected); } - // A slug is an exact directory lookup and must not be blocked by an - // unrelated invalid fixture. - if let Some(dir) = dirs - .iter() - .find(|dir| dir.file_name().and_then(|name| name.to_str()) == Some(selector)) - { - let fixture = ScenarioFixture::load(dir)?; - reject_duplicate_selected_id(&dirs, dir, &fixture.scenario.id)?; - return Ok(vec![fixture]); + // Slug and id selection compile only the requested scenario, so an + // unrelated fixture that fails compilation cannot block it. Malformed + // unrelated fixtures remain the responsibility of `validate --scenario + // all`. + if let Some(entry) = registered.iter().find(|entry| entry.slug == selector) { + return Ok(vec![ScenarioFixture::from_registered( + entry, + scenarios_root, + )?]); } - - // ID lookup reads only identities first, then compiles the one selected - // fixture. Malformed unrelated fixtures remain the responsibility of - // `validate --scenario all`. - let matching: Vec<&PathBuf> = dirs + if let Some(entry) = registered .iter() - .filter(|dir| { - read_scenario_id(dir) - .map(|id| id == selector) - .unwrap_or(false) - }) - .collect(); - match matching.as_slice() { - [dir] => return Ok(vec![ScenarioFixture::load(dir)?]), - [] => {} - duplicates => { - anyhow::bail!( - "scenario id {selector:?} is duplicated in {}", - duplicates - .iter() - .map(|dir| dir.display().to_string()) - .collect::>() - .join(", ") - ); - } + .find(|entry| entry.authored.id == selector) + { + return Ok(vec![ScenarioFixture::from_registered( + entry, + scenarios_root, + )?]); } anyhow::bail!("no scenario matches selector {selector:?}") } -/// Refuse `init` when an authored scenario already owns the requested id. -pub fn ensure_scenario_id_available( - scenarios_root: &Path, - scenario_id: &str, -) -> anyhow::Result<()> { - crate::types::scenario::validate_scenario_id(scenario_id)?; - for dir in scenario_directories(scenarios_root)? { - if read_scenario_id(&dir)? == scenario_id { +/// Identity checks read only authored data, never compile, so they cannot be +/// masked by an unrelated compilation failure. +fn reject_duplicate_identities(registered: &[RegisteredScenario]) -> anyhow::Result<()> { + let mut slugs = std::collections::BTreeMap::new(); + let mut ids = std::collections::BTreeMap::new(); + for entry in registered { + if let Some(previous) = slugs.insert(entry.slug.clone(), entry.authored.id.clone()) { anyhow::bail!( - "scenario id {scenario_id:?} already exists in {}", - dir.display() + "duplicate scenario slug {:?} (ids {:?} and {:?})", + entry.slug, + previous, + entry.authored.id ); } - } - Ok(()) -} - -fn scenario_directories(scenarios_root: &Path) -> anyhow::Result> { - let mut dirs = Vec::new(); - for entry in std::fs::read_dir(scenarios_root) - .with_context(|| format!("reading {}", scenarios_root.display()))? - { - let entry = entry - .with_context(|| format!("reading an entry under {}", scenarios_root.display()))?; - let path = entry.path(); - if path.join("scenario.yaml").is_file() { - dirs.push(path); + if let Some(previous) = ids.insert(entry.authored.id.clone(), entry.slug.clone()) { + anyhow::bail!( + "duplicate scenario id {:?} in {:?} and {:?}", + entry.authored.id, + previous, + entry.slug + ); } } - dirs.sort(); - Ok(dirs) -} - -#[derive(serde::Deserialize)] -struct ScenarioIdentity { - id: String, -} - -fn read_scenario_id(dir: &Path) -> anyhow::Result { - let path = dir.join("scenario.yaml"); - let source = std::fs::read_to_string(&path) - .with_context(|| format!("reading scenario identity from {}", path.display()))?; - let identity: ScenarioIdentity = serde_yaml::from_str(&source) - .with_context(|| format!("parsing scenario identity from {}", path.display()))?; - Ok(identity.id) -} - -fn reject_duplicate_selected_id( - dirs: &[PathBuf], - selected_dir: &Path, - selected_id: &str, -) -> anyhow::Result<()> { - let duplicates = dirs - .iter() - .filter(|dir| dir.as_path() != selected_dir) - .filter(|dir| { - read_scenario_id(dir) - .map(|id| id == selected_id) - .unwrap_or(false) - }) - .map(|dir| dir.display().to_string()) - .collect::>(); - if !duplicates.is_empty() { - anyhow::bail!( - "scenario id {selected_id:?} from {} is duplicated in {}", - selected_dir.display(), - duplicates.join(", ") - ); - } Ok(()) } diff --git a/harness/evals/integration/src/fixtures/loading.rs b/harness/evals/integration/src/fixtures/loading.rs index 5403fb545..b51e58fd8 100644 --- a/harness/evals/integration/src/fixtures/loading.rs +++ b/harness/evals/integration/src/fixtures/loading.rs @@ -1,16 +1,15 @@ -use std::path::{Path, PathBuf}; +use std::path::Path; use anyhow::Context; use super::script_validation::validate_script; use crate::expand::{compile_scenario, CompiledFixtureV1}; -use crate::types::scenario::{AuthoredScenarioV1, CompiledScenarioV1}; +use crate::scenarios::RegisteredScenario; +use crate::types::scenario::CompiledScenarioV1; use crate::types::script::RouterScriptV1; #[derive(Debug, Clone)] pub struct ScenarioFixture { - pub dir: PathBuf, - pub authored: AuthoredScenarioV1, pub scenario: CompiledScenarioV1, pub script: RouterScriptV1, /// Compiled shared golden plus inferred session/policy aid. @@ -18,17 +17,12 @@ pub struct ScenarioFixture { } impl ScenarioFixture { - pub fn load(dir: &Path) -> anyhow::Result { - let scenario_path = dir.join("scenario.yaml"); - let scenario: AuthoredScenarioV1 = serde_yaml::from_str( - &std::fs::read_to_string(&scenario_path) - .with_context(|| format!("reading {}", scenario_path.display()))?, - ) - .with_context(|| format!("parsing {}", scenario_path.display()))?; - - let scenarios_root = dir.parent().with_context(|| { - format!("scenario directory {} has no scenarios root", dir.display()) - })?; + /// Compile one registered scenario against the shared system prompt in + /// `scenarios_root`. + pub fn from_registered( + entry: &RegisteredScenario, + scenarios_root: &Path, + ) -> anyhow::Result { let prompt_path = scenarios_root.join("system-prompt.txt"); let system_prompt_base = std::fs::read_to_string(&prompt_path) .with_context(|| format!("reading {}", prompt_path.display()))?; @@ -36,12 +30,10 @@ impl ScenarioFixture { scenario: compiled, script, system_prompt_template, - } = compile_scenario(&scenario, &system_prompt_base) - .with_context(|| format!("compiling {}", scenario_path.display()))?; + } = compile_scenario(&entry.authored, &system_prompt_base) + .with_context(|| format!("compiling scenario {}", entry.slug))?; let fixture = ScenarioFixture { - dir: dir.to_path_buf(), - authored: scenario, scenario: compiled, script, system_prompt_template, diff --git a/harness/evals/integration/src/fixtures/tests.rs b/harness/evals/integration/src/fixtures/tests.rs index 8eb48fe0c..ea2d43b62 100644 --- a/harness/evals/integration/src/fixtures/tests.rs +++ b/harness/evals/integration/src/fixtures/tests.rs @@ -1,5 +1,5 @@ use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; +use std::path::Path; use serde_json::json; @@ -243,98 +243,93 @@ fn function_call_end_cannot_hide_incorrect_argument_deltas() { assert!(error_chain(script).contains("deltas disagree")); } -#[test] -fn all_selector_with_no_runnable_scenario_is_an_error() { - let empty = tempfile::tempdir().unwrap(); - let error = scenario_fixtures(empty.path(), "all", false).unwrap_err(); - assert!(format!("{error:#}").contains("no non-quarantined scenario")); +fn registered_text_scenario(slug: &str, id: &str) -> crate::scenarios::RegisteredScenario { + crate::scenarios::RegisteredScenario { + slug: slug.to_string(), + authored: crate::expand::scenario_template( + id, + "A fixture selection test.", + crate::expand::ScenarioTemplateKind::Text, + ), + } } -fn write_text_scenario(root: &Path, slug: &str, id: &str) { - let dir = root.join(slug); - std::fs::create_dir(&dir).unwrap(); - let authored = crate::expand::scenario_template( - id, - "A fixture selection test.", - crate::expand::ScenarioTemplateKind::Text, - ); - std::fs::write( - dir.join("scenario.yaml"), - crate::expand::render_authored_yaml(&authored).unwrap(), - ) - .unwrap(); +/// Selection needs only the shared system prompt on disk. +fn prompt_root() -> tempfile::TempDir { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("system-prompt.txt"), "base").unwrap(); + root } #[test] -fn explicit_selection_isolated_from_unrelated_invalid_fixtures() { - let root = tempfile::tempdir().unwrap(); - std::fs::write(root.path().join("system-prompt.txt"), "base").unwrap(); - let invalid = root.path().join("a-invalid"); - std::fs::create_dir(&invalid).unwrap(); - std::fs::write(invalid.join("scenario.yaml"), "not: [valid").unwrap(); - write_text_scenario(root.path(), "z-selected", "C-E2E-SELECTED"); +fn all_selector_with_no_runnable_scenario_is_an_error() { + let root = prompt_root(); - assert_eq!( - scenario_fixtures(root.path(), "z-selected", false).unwrap()[0] - .scenario - .id, - "C-E2E-SELECTED" - ); - assert_eq!( - scenario_fixtures(root.path(), "C-E2E-SELECTED", false).unwrap()[0] - .scenario - .id, - "C-E2E-SELECTED" - ); - assert!(scenario_fixtures(root.path(), "all", true).is_err()); + let empty_error = super::discovery::select_fixtures(Vec::new(), root.path(), "all", true) + .unwrap_err(); + assert!(format!("{empty_error:#}").contains("matched no registered scenario")); + + let mut quarantined = registered_text_scenario("only-quarantined", "C-E2E-Q"); + quarantined.authored.quarantine = true; + let error = super::discovery::select_fixtures(vec![quarantined], root.path(), "all", false) + .unwrap_err(); + assert!(format!("{error:#}").contains("no non-quarantined registered scenario")); } #[test] -fn duplicate_ids_are_rejected_and_unavailable_to_init() { - let root = tempfile::tempdir().unwrap(); - std::fs::write(root.path().join("system-prompt.txt"), "base").unwrap(); - write_text_scenario(root.path(), "first", "C-E2E-DUPLICATE"); - write_text_scenario(root.path(), "second", "C-E2E-DUPLICATE"); - - let all_error = scenario_fixtures(root.path(), "all", true).unwrap_err(); - assert!(format!("{all_error:#}").contains("duplicate scenario id")); - let id_error = scenario_fixtures(root.path(), "C-E2E-DUPLICATE", true).unwrap_err(); - assert!(format!("{id_error:#}").contains("duplicated")); - let slug_error = scenario_fixtures(root.path(), "first", true).unwrap_err(); - assert!(format!("{slug_error:#}").contains("duplicated")); - let init_error = ensure_scenario_id_available(root.path(), "C-E2E-DUPLICATE").unwrap_err(); - assert!(format!("{init_error:#}").contains("already exists")); +fn explicit_selection_isolated_from_unrelated_invalid_fixtures() { + let root = prompt_root(); + // Well-typed but semantically broken: the function call resolves no + // registered alias, so only compilation can reject it. + let mut invalid = registered_text_scenario("a-invalid", "C-E2E-INVALID"); + invalid.authored.router.generations[0].reply = + crate::types::scenario::RouterReplyV1::FunctionCall { + id: None, + function: "missing".to_string(), + arguments: json!({}), + usage: None, + }; + let registered = vec![ + invalid, + registered_text_scenario("z-selected", "C-E2E-SELECTED"), + ]; + + let by_slug = + super::discovery::select_fixtures(registered.clone(), root.path(), "z-selected", false) + .unwrap(); + assert_eq!(by_slug[0].scenario.id, "C-E2E-SELECTED"); + let by_id = + super::discovery::select_fixtures(registered.clone(), root.path(), "C-E2E-SELECTED", false) + .unwrap(); + assert_eq!(by_id[0].scenario.id, "C-E2E-SELECTED"); + assert!(super::discovery::select_fixtures(registered, root.path(), "all", true).is_err()); } #[test] -fn checked_in_scenarios_are_single_file_and_compile() { - fn files_under(dir: &Path, files: &mut Vec) { - for entry in std::fs::read_dir(dir).unwrap() { - let path = entry.unwrap().path(); - if path.is_dir() { - files_under(&path, files); - } else { - files.push(path); - } - } - } - - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("scenarios"); - let fixtures = scenario_fixtures(&root, "all", true).unwrap(); - assert!( - !fixtures.is_empty(), - "expected at least one checked-in scenario" - ); - for fixture in fixtures { - let mut files = Vec::new(); - files_under(&fixture.dir, &mut files); - assert_eq!( - files, - vec![fixture.dir.join("scenario.yaml")], - "{} must contain only scenario.yaml", - fixture.dir.display() +fn duplicate_identities_are_rejected_for_every_selector() { + let root = prompt_root(); + let registered = vec![ + registered_text_scenario("first", "C-E2E-DUPLICATE"), + registered_text_scenario("second", "C-E2E-DUPLICATE"), + ]; + + for selector in ["all", "C-E2E-DUPLICATE", "first"] { + let error = + super::discovery::select_fixtures(registered.clone(), root.path(), selector, true) + .unwrap_err(); + assert!( + format!("{error:#}").contains("duplicate scenario id"), + "{selector}: {error:#}" ); } + + let duplicate_slug = vec![ + registered_text_scenario("twice", "C-E2E-FIRST"), + registered_text_scenario("twice", "C-E2E-SECOND"), + ]; + let error = super::discovery::select_fixtures(duplicate_slug, root.path(), "all", true) + .unwrap_err(); + assert!(format!("{error:#}").contains("duplicate scenario slug")); } #[test] diff --git a/harness/evals/integration/src/lib.rs b/harness/evals/integration/src/lib.rs index 1e0aab0a6..52a8b3592 100644 --- a/harness/evals/integration/src/lib.rs +++ b/harness/evals/integration/src/lib.rs @@ -18,6 +18,7 @@ pub mod readiness; pub mod recorder; pub mod runtime; pub mod scenario; +pub mod scenarios; pub mod scripted_router; pub mod services; pub mod stack; diff --git a/harness/evals/integration/src/main.rs b/harness/evals/integration/src/main.rs index 0ad4cda2a..e038e7542 100644 --- a/harness/evals/integration/src/main.rs +++ b/harness/evals/integration/src/main.rs @@ -2,14 +2,9 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use anyhow::Context; -use clap::{Args, Parser, Subcommand, ValueEnum}; -use harness_integration::expand::{ - compile_scenario, render_authored_yaml, render_compiled, scenario_template, - ScenarioTemplateKind, -}; -use harness_integration::fixtures::{ - ensure_scenario_id_available, scenario_fixtures, ScenarioFixture, -}; +use clap::{Args, Parser, Subcommand}; +use harness_integration::expand::render_compiled; +use harness_integration::fixtures::{scenario_fixtures, ScenarioFixture}; use harness_integration::scenario::run_scenario; use harness_integration::stack::StackBins; use harness_integration::types::scenario::Classification; @@ -30,12 +25,11 @@ struct Cli { enum Command { /// Run one scenario or every non-quarantined scenario. Run(RunArgs), - /// Compile and validate fixtures without booting a stack. + /// Compile and validate every registered scenario without booting a + /// stack. Validate(SelectionArgs), /// Print one deterministic, fully expanded compiled scenario. Render(RenderArgs), - /// Create one valid single-file scenario without overwriting files. - Init(InitArgs), } #[derive(Debug, Args)] @@ -71,62 +65,24 @@ struct RunArgs { #[derive(Debug, Clone, Args)] struct SelectionArgs { - /// Scenario id, scenario directory name, or `all`. + /// Scenario id, scenario slug, or `all`. #[arg(long, default_value = "all")] scenario: String, - /// Directory containing the checked-in scenarios. + /// Directory containing the shared system prompt template. #[arg(long, default_value = DEFAULT_SCENARIOS_DIR)] scenarios_dir: PathBuf, } #[derive(Debug, Args)] struct RenderArgs { - /// Scenario id or directory name. + /// Scenario id or slug. scenario: String, #[arg(long, default_value = DEFAULT_SCENARIOS_DIR)] scenarios_dir: PathBuf, } -#[derive(Debug, Args)] -struct InitArgs { - #[arg(long)] - id: String, - - /// Directory slug created below the scenarios directory. - #[arg(long)] - name: String, - - #[arg(long)] - description: String, - - #[arg(long, value_enum)] - kind: TemplateKind, - - #[arg(long, default_value = DEFAULT_SCENARIOS_DIR)] - scenarios_dir: PathBuf, -} - -#[derive(Debug, Clone, Copy, ValueEnum)] -enum TemplateKind { - Text, - Function, - Hook, - Crash, -} - -impl From for ScenarioTemplateKind { - fn from(value: TemplateKind) -> Self { - match value { - TemplateKind::Text => Self::Text, - TemplateKind::Function => Self::Function, - TemplateKind::Hook => Self::Hook, - TemplateKind::Crash => Self::Crash, - } - } -} - fn parse_worker_bin(raw: &str) -> Result<(String, PathBuf), String> { let (name, path) = raw .split_once('=') @@ -164,7 +120,6 @@ async fn dispatch(cli: Cli) -> i32 { Command::Run(args) => return run(args).await, Command::Validate(args) => validate(args), Command::Render(args) => render(args), - Command::Init(args) => init(args), }; match result { Ok(message) => { @@ -289,76 +244,6 @@ fn render(args: RenderArgs) -> anyhow::Result { render_compiled(&fixture.compiled()) } -fn init(args: InitArgs) -> anyhow::Result { - validate_slug(&args.name)?; - anyhow::ensure!(!args.id.trim().is_empty(), "--id must not be empty"); - anyhow::ensure!( - !args.description.trim().is_empty(), - "--description must not be empty" - ); - - let target = args.scenarios_dir.join(&args.name); - anyhow::ensure!( - !target.exists(), - "refusing to overwrite existing scenario directory {}", - target.display() - ); - let authored = scenario_template(&args.id, &args.description, args.kind.into()); - let prompt_path = args.scenarios_dir.join("system-prompt.txt"); - let prompt = std::fs::read_to_string(&prompt_path) - .with_context(|| format!("reading shared prompt {}", prompt_path.display()))?; - compile_scenario(&authored, &prompt).context("generated scenario is invalid")?; - ensure_scenario_id_available(&args.scenarios_dir, &args.id)?; - let yaml = render_authored_yaml(&authored)?; - - let scenario_path = write_scenario_atomically(&target, &yaml)?; - Ok(format!("created {}", scenario_path.display())) -} - -fn write_scenario_atomically(target: &Path, yaml: &str) -> anyhow::Result { - let parent = target - .parent() - .context("scenario target has no parent directory")?; - let name = target - .file_name() - .and_then(|name| name.to_str()) - .context("scenario target name is not UTF-8")?; - let temporary = parent.join(format!(".{name}.{}.tmp", uuid::Uuid::new_v4().simple())); - std::fs::create_dir(&temporary) - .with_context(|| format!("creating temporary scenario {}", temporary.display()))?; - let temporary_file = temporary.join("scenario.yaml"); - let outcome = std::fs::write(&temporary_file, yaml) - .with_context(|| format!("writing {}", temporary_file.display())) - .and_then(|()| { - std::fs::rename(&temporary, target).with_context(|| { - format!( - "publishing temporary scenario {} as {}", - temporary.display(), - target.display() - ) - }) - }); - if let Err(error) = outcome { - let _ = std::fs::remove_dir_all(&temporary); - return Err(error); - } - Ok(target.join("scenario.yaml")) -} - -fn validate_slug(slug: &str) -> anyhow::Result<()> { - let valid = !slug.is_empty() - && slug != "." - && slug != ".." - && slug - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')); - anyhow::ensure!( - valid, - "--name must contain only ASCII letters, digits, '-' or '_'" - ); - Ok(()) -} - fn load_fixtures( selection: &SelectionArgs, include_quarantined: bool, @@ -418,16 +303,6 @@ fn resolve_bins(args: &RunArgs) -> anyhow::Result { mod tests { use super::*; - #[test] - fn scenario_names_cannot_escape_the_root() { - for invalid in ["", ".", "..", "../escape", "nested/name", "with space"] { - assert!(validate_slug(invalid).is_err(), "{invalid:?}"); - } - for valid in ["streamed-text", "case_507", "C505"] { - validate_slug(valid).unwrap(); - } - } - #[test] fn repeat_count_must_be_positive() { assert_eq!(parse_repeat("1").unwrap(), 1); diff --git a/harness/evals/integration/src/scenarios/builder.rs b/harness/evals/integration/src/scenarios/builder.rs new file mode 100644 index 000000000..8bdb51d56 --- /dev/null +++ b/harness/evals/integration/src/scenarios/builder.rs @@ -0,0 +1,433 @@ +//! Typed builders for authored scenarios. +//! +//! Builders produce data only: every method fills a field on the authored +//! structs in [`crate::types::scenario`] and returns the value for further +//! chaining. A builder that derives scenario content from control flow is +//! rejected in review, under the same rule that forbids a second +//! orchestration language in the authored layer. +//! +//! Defaults mirror the deterministic compiler defaults: a text reply without +//! chunks emits one terminal `done` frame, faults interrupt the first target +//! call after 1500 ms, and releases target `call-1`. + +use serde_json::Value; + +use crate::types::frames::Usage; +use crate::types::scenario::{ + AuthoredScenarioV1, ExpectationsV1, FaultKind, FaultV1, FunctionResultExpectationV1, + GenerationMatchOverridesV1, ReleaseActionV1, ReleaseV1, RouterReplyV1, ScenarioFunctionV1, + ScenarioGenerationV1, ScenarioRouterV1, ScenarioSendV1, TargetCallsExpectationV1, + TriggerBindingSpecV1, TriggerKindV1, +}; +use crate::types::script::{JsonMatcherV1, SchemaVersion1}; + +/// Authoring name for the scenario root. Scenario modules return this type. +pub type AuthoredScenario = AuthoredScenarioV1; + +impl AuthoredScenarioV1 { + /// A new scenario with an empty send. Chain [`Self::send`] before + /// registering; the registry tests reject an empty message. + pub fn new(id: &str, description: &str) -> Self { + Self { + schema_version: SchemaVersion1::V1, + id: id.to_string(), + description: description.to_string(), + quarantine: false, + send: Send::message(""), + functions: Default::default(), + router: ScenarioRouterV1 { + model: None, + generations: Vec::new(), + }, + bindings: Vec::new(), + release: None, + fault: None, + timeouts: Default::default(), + expect: Default::default(), + } + } + + /// Exclude this scenario from ordinary `all` runs; `validate` and + /// explicit selection still include it. + pub fn quarantine(mut self) -> Self { + self.quarantine = true; + self + } + + pub fn send(mut self, send: ScenarioSendV1) -> Self { + self.send = send; + self + } + + /// Register a controlled function under its alias. The compiler expands + /// aliases to `{{run_id}}::`. + pub fn function(mut self, alias: &str, function: ScenarioFunctionV1) -> Self { + self.functions.insert(alias.to_string(), function); + self + } + + pub fn generation(mut self, generation: impl Into) -> Self { + self.router.generations.push(generation.into()); + self + } + + pub fn binding(mut self, binding: TriggerBindingSpecV1) -> Self { + self.bindings.push(binding); + self + } + + /// Release the deterministic first call (`call-1`) once it is held. + pub fn release(mut self, action: ReleaseActionV1) -> Self { + self.release = Some(ReleaseV1 { + function_call_id: "call-1".to_string(), + action, + }); + self + } + + pub fn fault(mut self, fault: FaultV1) -> Self { + self.fault = Some(fault); + self + } + + pub fn readiness_timeout_ms(mut self, readiness_ms: u64) -> Self { + self.timeouts.readiness_ms = readiness_ms; + self + } + + pub fn scenario_timeout_ms(mut self, scenario_ms: u64) -> Self { + self.timeouts.scenario_ms = scenario_ms; + self + } + + pub fn teardown_timeout_ms(mut self, teardown_ms: u64) -> Self { + self.timeouts.teardown_ms = teardown_ms; + self + } + + pub fn expect(mut self, expect: ExpectationsV1) -> Self { + self.expect = expect; + self + } +} + +/// Entry point for [`ScenarioSendV1`]. Shadows the marker trait name inside +/// scenario modules on purpose; those modules contain data only. +pub struct Send; + +impl Send { + pub fn message(message: &str) -> ScenarioSendV1 { + ScenarioSendV1 { + message: message.to_string(), + allow: None, + idempotency_key: None, + } + } +} + +impl ScenarioSendV1 { + /// Allowed function aliases. Omitting the call keeps every exposed + /// function; `allow([])` disables function dispatch. + pub fn allow(mut self, aliases: [&str; N]) -> Self { + self.allow = Some(aliases.iter().map(|alias| alias.to_string()).collect()); + self + } + + pub fn idempotency_key(mut self, key: &str) -> Self { + self.idempotency_key = Some(key.to_string()); + self + } +} + +/// Authoring name for [`ScenarioFunctionV1`]. +pub type Function = ScenarioFunctionV1; + +impl ScenarioFunctionV1 { + /// A controlled function exposed to the model. `request_schema` must be + /// a JSON object. + pub fn new(description: &str, request_schema: Value, response: Value) -> Self { + Self { + description: description.to_string(), + request_schema: request_schema + .as_object() + .cloned() + .expect("request_schema must be a JSON object"), + response, + expose: true, + } + } + + /// Hook-only controlled functions are never exposed to the model. + pub fn hidden(mut self) -> Self { + self.expose = false; + self + } +} + +/// Entry point for typed router replies. +pub struct Reply; + +impl Reply { + pub fn text(text: &str) -> TextReply { + TextReply { + text: text.to_string(), + chunks: Vec::new(), + usage: None, + } + } + + /// A function call against a registered alias, closed as + /// `call-` by the compiler. + pub fn function_call(function: &str, arguments: Value) -> FunctionCallReply { + FunctionCallReply { + function: function.to_string(), + arguments, + usage: None, + } + } +} + +pub struct TextReply { + text: String, + chunks: Vec, + usage: Option, +} + +impl TextReply { + /// Non-empty chunks produce the complete streaming frame sequence; + /// without chunks the reply is one terminal `done` frame. + pub fn chunks(mut self, chunks: [&str; N]) -> Self { + self.chunks = chunks.iter().map(|chunk| chunk.to_string()).collect(); + self + } + + pub fn usage(mut self, input: u64, output: u64) -> Self { + self.usage = Some(input_output_usage(input, output)); + self + } + + pub fn match_overrides(self, overrides: GenerationMatchOverridesV1) -> ScenarioGenerationV1 { + with_overrides(self.into(), overrides) + } +} + +impl From for ScenarioGenerationV1 { + fn from(reply: TextReply) -> Self { + plain_generation(RouterReplyV1::Text { + text: reply.text, + chunks: reply.chunks, + usage: reply.usage, + }) + } +} + +pub struct FunctionCallReply { + function: String, + arguments: Value, + usage: Option, +} + +impl FunctionCallReply { + pub fn usage(mut self, input: u64, output: u64) -> Self { + self.usage = Some(input_output_usage(input, output)); + self + } + + pub fn match_overrides(self, overrides: GenerationMatchOverridesV1) -> ScenarioGenerationV1 { + with_overrides(self.into(), overrides) + } +} + +impl From for ScenarioGenerationV1 { + fn from(reply: FunctionCallReply) -> Self { + plain_generation(RouterReplyV1::FunctionCall { + id: None, + function: reply.function, + arguments: reply.arguments, + usage: reply.usage, + }) + } +} + +fn plain_generation(reply: RouterReplyV1) -> ScenarioGenerationV1 { + ScenarioGenerationV1 { + reply, + match_overrides: Default::default(), + } +} + +fn with_overrides( + mut generation: ScenarioGenerationV1, + overrides: GenerationMatchOverridesV1, +) -> ScenarioGenerationV1 { + generation.match_overrides = overrides; + generation +} + +fn input_output_usage(input: u64, output: u64) -> Usage { + Usage { + input: Some(input), + output: Some(output), + cache_read: None, + cache_write: None, + reasoning: None, + cost_usd: None, + } +} + +/// Entry point for [`TriggerBindingSpecV1`]. +pub struct Binding; + +impl Binding { + pub fn hook_pre_trigger( + function: &str, + functions: [&str; N], + priority: i64, + ) -> TriggerBindingSpecV1 { + TriggerBindingSpecV1 { + trigger: TriggerKindV1::HookPreTrigger, + function: function.to_string(), + functions: functions.iter().map(|alias| alias.to_string()).collect(), + priority, + } + } +} + +/// Entry point for [`FaultV1`], mirroring the deterministic defaults the +/// YAML layer used to apply. +pub struct Fault; + +impl Fault { + pub fn engine_sigkill() -> FaultV1 { + FaultV1 { + kind: FaultKind::EngineSigkill, + function: None, + after_target_calls: 1, + restart_delay_ms: 1_500, + } + } +} + +impl FaultV1 { + pub fn function(mut self, alias: &str) -> Self { + self.function = Some(alias.to_string()); + self + } + + pub fn after_target_calls(mut self, calls: u64) -> Self { + self.after_target_calls = calls; + self + } + + pub fn restart_delay_ms(mut self, delay_ms: u64) -> Self { + self.restart_delay_ms = delay_ms; + self + } +} + +/// Authoring name for [`ExpectationsV1`]. +pub type Expect = ExpectationsV1; + +impl ExpectationsV1 { + pub fn new() -> Self { + Self::default() + } + + pub fn message_counts(mut self, user: u64, assistant: u64, function_result: u64) -> Self { + self.message_counts = Some(crate::types::scenario::MessageCountsExpectationV1 { + user, + assistant, + function_result, + }); + self + } + + pub fn assistant_text(mut self, text: &str) -> Self { + self.assistant_text = Some(text.to_string()); + self + } + + pub fn function_result(mut self, result: FunctionResultExpectationV1) -> Self { + self.function_results.push(result); + self + } + + pub fn calls_closed(mut self) -> Self { + self.calls_closed = true; + self + } + + pub fn call(mut self, call: TargetCallsExpectationV1) -> Self { + self.calls.push(call); + self + } +} + +/// Entry point for [`FunctionResultExpectationV1`]. +pub struct FunctionResult; + +impl FunctionResult { + /// A durable function result closing the given call id, with no further + /// constraints until chained. + pub fn closing(function_call_id: &str) -> FunctionResultExpectationV1 { + FunctionResultExpectationV1 { + function_call_id: function_call_id.to_string(), + function: None, + content: None, + is_error: None, + } + } +} + +impl FunctionResultExpectationV1 { + pub fn function(mut self, alias: &str) -> Self { + self.function = Some(alias.to_string()); + self + } + + pub fn content(mut self, content: Vec) -> Self { + self.content = Some(content); + self + } + + pub fn is_error(mut self, is_error: bool) -> Self { + self.is_error = Some(is_error); + self + } +} + +/// Entry point for [`TargetCallsExpectationV1`]. +pub struct TargetCall; + +impl TargetCall { + pub fn counted(function: &str, count: u64) -> TargetCallsExpectationV1 { + TargetCallsExpectationV1 { + function: function.to_string(), + count, + payload: None, + payload_subset: None, + } + } +} + +impl TargetCallsExpectationV1 { + pub fn payload(mut self, payload: Value) -> Self { + self.payload = Some(payload); + self + } + + pub fn payload_subset(mut self, payload_subset: Value) -> Self { + self.payload_subset = Some(payload_subset); + self + } +} + +pub fn regex(pattern: &str) -> JsonMatcherV1 { + JsonMatcherV1::Regex { + pattern: pattern.to_string(), + } +} + +pub fn present() -> JsonMatcherV1 { + JsonMatcherV1::Present +} diff --git a/harness/evals/integration/src/scenarios/crash_recovery_507.rs b/harness/evals/integration/src/scenarios/crash_recovery_507.rs new file mode 100644 index 000000000..9020be60e --- /dev/null +++ b/harness/evals/integration/src/scenarios/crash_recovery_507.rs @@ -0,0 +1,58 @@ +//! C-E2E-507 — crash recovery closes the interrupted function call. +//! +//! Reproduction of . + +use serde_json::json; + +use crate::types::scenario::GenerationMatchOverridesV1; + +use super::builder::*; + +pub(super) fn scenario() -> AuthoredScenario { + AuthoredScenario::new( + "C-E2E-507", + "An engine crash during a dispatched function call must not leave the call dangling or the session unusable.", + ) + .quarantine() + .send(Send::message("Call the recorder once.")) + .function( + "record", + Function::new( + "Record one integration fixture value.", + json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + }), + json!({ + "content": [{ "type": "text", "text": "recorded" }], + "is_error": false + }), + ), + ) + .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) + // Recovery can legitimately reconstruct the second request differently; + // this reproduction grades the durable outcome instead. + .generation( + Reply::text("recovered") + .usage(20, 2) + .match_overrides(GenerationMatchOverridesV1 { + request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), + system_prompt: Some(present()), + messages: Some(present()), + tools: Some(present()), + ..Default::default() + }), + ) + .fault(Fault::engine_sigkill()) + .scenario_timeout_ms(120_000) + .expect( + Expect::new() + .message_counts(1, 2, 1) + .assistant_text("recovered") + .calls_closed() + .function_result(FunctionResult::closing("call-1")) + .call(TargetCall::counted("record", 1).payload(json!({ "value": "expected" }))), + ) +} diff --git a/harness/evals/integration/src/scenarios/exactly_once_function.rs b/harness/evals/integration/src/scenarios/exactly_once_function.rs new file mode 100644 index 000000000..2646f7601 --- /dev/null +++ b/harness/evals/integration/src/scenarios/exactly_once_function.rs @@ -0,0 +1,43 @@ +//! C-E2E-002 — an allow-listed function executes exactly once. + +use serde_json::json; + +use super::builder::*; + +pub(super) fn scenario() -> AuthoredScenario { + AuthoredScenario::new( + "C-E2E-002", + "An allow-listed native function executes exactly once with a durable result.", + ) + .send(Send::message("Call the recorder once.")) + .function( + "record", + Function::new( + "Record one integration fixture value.", + json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + }), + json!({ + "content": [{ "type": "text", "text": "recorded" }], + "is_error": false + }), + ), + ) + .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) + .generation(Reply::text("recorded once").usage(18, 2)) + .expect( + Expect::new() + .message_counts(1, 2, 1) + .assistant_text("recorded once") + .function_result( + FunctionResult::closing("call-1") + .function("record") + .content(vec![json!({ "type": "text", "text": "recorded" })]) + .is_error(false), + ) + .call(TargetCall::counted("record", 1).payload(json!({ "value": "expected" }))), + ) +} diff --git a/harness/evals/integration/src/scenarios/hold_mutation_505.rs b/harness/evals/integration/src/scenarios/hold_mutation_505.rs new file mode 100644 index 000000000..fd97b6b2a --- /dev/null +++ b/harness/evals/integration/src/scenarios/hold_mutation_505.rs @@ -0,0 +1,77 @@ +//! C-E2E-505 — a holding hook's mutation reaches the released call. +//! +//! Reproduction of . + +use serde_json::json; + +use crate::types::scenario::GenerationMatchOverridesV1; + +use super::builder::*; + +pub(super) fn scenario() -> AuthoredScenario { + AuthoredScenario::new( + "C-E2E-505", + "A pre-trigger hook that holds and mutates must apply its mutation to the released call.", + ) + .quarantine() + .send(Send::message("Call the recorder once.")) + .function( + "record", + Function::new( + "Record one integration fixture value.", + json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + }), + json!({ + "content": [{ "type": "text", "text": "recorded" }], + "is_error": false + }), + ), + ) + .function( + "hook-gate", + Function::new( + "Hold the call and stamp approval context onto its arguments.", + json!({ "type": "object" }), + json!({ + "decision": "hold", + "mutations": { "arguments": { "value": "expected+approved" } } + }), + ) + .hidden(), + ) + .binding(Binding::hook_pre_trigger("hook-gate", ["record"], 10)) + .release(crate::types::scenario::ReleaseActionV1::Execute) + .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) + .generation( + Reply::text("approved and recorded") + .usage(20, 3) + .match_overrides(GenerationMatchOverridesV1 { + request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), + system_prompt: Some(present()), + messages: Some(present()), + tools: Some(present()), + ..Default::default() + }), + ) + .expect( + Expect::new() + .calls_closed() + .call( + TargetCall::counted("record", 1).payload(json!({ "value": "expected+approved" })), + ) + .call( + TargetCall::counted("hook-gate", 1).payload_subset(json!({ + "point": "pre_trigger", + "call": { + "id": "call-1", + "function_id": "{{run_id}}::record", + "arguments": { "value": "expected" } + } + })), + ), + ) +} diff --git a/harness/evals/integration/src/scenarios/hook_held_release_506.rs b/harness/evals/integration/src/scenarios/hook_held_release_506.rs new file mode 100644 index 000000000..eb180c33a --- /dev/null +++ b/harness/evals/integration/src/scenarios/hook_held_release_506.rs @@ -0,0 +1,95 @@ +//! C-E2E-506 — released held calls retain hook-mutated arguments. +//! +//! Reproduction of . + +use serde_json::json; + +use crate::types::scenario::GenerationMatchOverridesV1; + +use super::builder::*; + +pub(super) fn scenario() -> AuthoredScenario { + AuthoredScenario::new( + "C-E2E-506", + "A held call released for execution must run with the arguments produced by earlier hooks.", + ) + .quarantine() + .send(Send::message("Call the recorder once.")) + .function( + "record", + Function::new( + "Record one integration fixture value.", + json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + }), + json!({ + "content": [{ "type": "text", "text": "recorded" }], + "is_error": false + }), + ), + ) + .function( + "hook-mutate", + Function::new( + "Inject validated scope into the arguments.", + json!({ "type": "object" }), + json!({ + "decision": "continue", + "mutations": { "arguments": { "value": "expected+scope" } } + }), + ) + .hidden(), + ) + .function( + "hook-hold", + Function::new( + "Hold every consulted call for explicit approval.", + json!({ "type": "object" }), + json!({ "decision": "hold" }), + ) + .hidden(), + ) + .binding(Binding::hook_pre_trigger("hook-mutate", ["record"], 10)) + .binding(Binding::hook_pre_trigger("hook-hold", ["record"], 20)) + .release(crate::types::scenario::ReleaseActionV1::Execute) + .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) + .generation( + Reply::text("released and recorded") + .usage(20, 3) + .match_overrides(GenerationMatchOverridesV1 { + request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), + system_prompt: Some(present()), + messages: Some(present()), + tools: Some(present()), + ..Default::default() + }), + ) + .expect( + Expect::new() + .calls_closed() + .call(TargetCall::counted("record", 1).payload(json!({ "value": "expected+scope" }))) + .call( + TargetCall::counted("hook-mutate", 1).payload_subset(json!({ + "point": "pre_trigger", + "call": { + "id": "call-1", + "function_id": "{{run_id}}::record", + "arguments": { "value": "expected" } + } + })), + ) + .call( + TargetCall::counted("hook-hold", 1).payload_subset(json!({ + "point": "pre_trigger", + "call": { + "id": "call-1", + "function_id": "{{run_id}}::record", + "arguments": { "value": "expected+scope" } + } + })), + ), + ) +} diff --git a/harness/evals/integration/src/scenarios/mod.rs b/harness/evals/integration/src/scenarios/mod.rs new file mode 100644 index 000000000..2a320851c --- /dev/null +++ b/harness/evals/integration/src/scenarios/mod.rs @@ -0,0 +1,97 @@ +//! Authored scenario modules and their registry. +//! +//! One Rust builder module per scenario: each `src/scenarios/.rs` +//! builds the authored scenario data through the typed builders in +//! [`builder`] and registers it in [`all`]. There is no YAML layer — the +//! authored shape is the runner's own data model, enforced at `cargo build`, +//! and it is never serialized. Compiled snapshots under `tests/snapshots/` +//! remain the review artifact. + +pub mod builder; + +mod crash_recovery_507; +mod exactly_once_function; +mod hold_mutation_505; +mod hook_held_release_506; +mod streamed_text; + +use crate::types::scenario::AuthoredScenarioV1; + +/// One authored scenario plus the stable slug used by `--scenario` selection +/// and the compiled-snapshot filename. +#[derive(Debug, Clone)] +pub struct RegisteredScenario { + pub slug: String, + pub authored: AuthoredScenarioV1, +} + +/// Every authored scenario, in stable slug order. +pub fn all() -> Vec { + vec![ + register("crash-recovery-507", crash_recovery_507::scenario()), + register("exactly-once-function", exactly_once_function::scenario()), + register("hold-mutation-505", hold_mutation_505::scenario()), + register("hook-held-release-506", hook_held_release_506::scenario()), + register("streamed-text", streamed_text::scenario()), + ] +} + +fn register(slug: &str, authored: AuthoredScenarioV1) -> RegisteredScenario { + RegisteredScenario { + slug: slug.to_string(), + authored, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expand::compile_scenario; + use crate::types::scenario::validate_scenario_id; + + /// Slugs double as snapshot filenames and `--scenario` selectors. + fn validate_slug(slug: &str) { + assert!( + !slug.is_empty() + && slug + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')), + "slug {slug:?} must contain only ASCII letters, digits, '-' or '_'" + ); + } + + #[test] + fn every_scenario_registers_exactly_once() { + let registered = all(); + assert!(!registered.is_empty()); + let mut slugs = std::collections::BTreeSet::new(); + let mut ids = std::collections::BTreeSet::new(); + for entry in ®istered { + validate_slug(&entry.slug); + validate_scenario_id(&entry.authored.id).unwrap(); + assert!( + slugs.insert(entry.slug.clone()), + "slug {:?} registered more than once", + entry.slug + ); + assert!( + ids.insert(entry.authored.id.clone()), + "scenario id {:?} registered more than once", + entry.authored.id + ); + } + } + + #[test] + fn every_scenario_compiles() { + for entry in all() { + assert!( + !entry.authored.send.message.is_empty(), + "{}: authored send message must not be empty", + entry.slug + ); + compile_scenario(&entry.authored, "base prompt\n") + .unwrap_or_else(|error| panic!("{} does not compile: {error:#}", entry.slug)); + } + } +} diff --git a/harness/evals/integration/src/scenarios/streamed_text.rs b/harness/evals/integration/src/scenarios/streamed_text.rs new file mode 100644 index 000000000..ae8810110 --- /dev/null +++ b/harness/evals/integration/src/scenarios/streamed_text.rs @@ -0,0 +1,21 @@ +//! C-E2E-001 — streamed text reaches durable completion. + +use super::builder::*; + +pub(super) fn scenario() -> AuthoredScenario { + AuthoredScenario::new( + "C-E2E-001", + "Streamed text reaches durable completion through the real queue and turn loop.", + ) + .send(Send::message("Return the fixture phrase.")) + .generation( + Reply::text("fixture complete") + .chunks(["fixture ", "complete"]) + .usage(8, 2), + ) + .expect( + Expect::new() + .message_counts(1, 1, 0) + .assistant_text("fixture complete"), + ) +} diff --git a/harness/evals/integration/src/types/scenario/authored.rs b/harness/evals/integration/src/types/scenario/authored.rs index 59176b40d..817816108 100644 --- a/harness/evals/integration/src/types/scenario/authored.rs +++ b/harness/evals/integration/src/types/scenario/authored.rs @@ -8,39 +8,30 @@ use crate::types::script::{JsonMatcherV1, ModelFixtureV1, SchemaVersion1}; use super::ExpectationsV1; -/// The single-file scenario authors maintain in `scenarios//scenario.yaml`. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +/// The authored scenario data built by the `src/scenarios` builder modules. +/// +/// This layer is code, never serialized: there is no schema pair to keep +/// synchronized and no round trip. `DeadlinesV1`, `ReleaseV1`, and +/// `FaultKind` below are shared with the compiled layer and keep their wire +/// derives. +#[derive(Debug, Clone, PartialEq)] pub struct AuthoredScenarioV1 { pub schema_version: SchemaVersion1, - #[schemars(length(min = 1, max = 128), regex(pattern = "^[A-Za-z0-9_-]+$"))] pub id: String, pub description: String, - #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub quarantine: bool, pub send: ScenarioSendV1, /// Alias → controlled function. Aliases are expanded to /// `{{run_id}}::` by the compiler. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - #[schemars(schema_with = "functions_schema")] pub functions: BTreeMap, pub router: ScenarioRouterV1, - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub bindings: Vec, - #[serde(skip_serializing_if = "Option::is_none")] pub release: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub fault: Option, - #[serde(default, skip_serializing_if = "DeadlinesV1::is_default")] pub timeouts: DeadlinesV1, - #[serde(default, skip_serializing_if = "ExpectationsV1::is_default")] pub expect: ExpectationsV1, } -/// Compatibility name retained for callers compiled against the first -/// single-file authoring API. New code should use [`AuthoredScenarioV1`]. -pub type IntegrationScenarioV1 = AuthoredScenarioV1; - /// Scenario ids are also artifact directory names, so keep them to one safe, /// portable path component. pub fn validate_scenario_id(id: &str) -> anyhow::Result<()> { @@ -56,71 +47,57 @@ pub fn validate_scenario_id(id: &str) -> anyhow::Result<()> { Ok(()) } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub struct ScenarioSendV1 { pub message: String, /// Allowed function aliases. Omitted means every function whose /// `expose` flag is true; an empty list disables function dispatch. - #[serde(skip_serializing_if = "Option::is_none")] pub allow: Option>, /// Omitted values are derived deterministically from the scenario id. - #[serde(skip_serializing_if = "Option::is_none")] pub idempotency_key: Option, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub struct ScenarioFunctionV1 { pub description: String, pub request_schema: serde_json::Map, pub response: serde_json::Value, /// Exposed to the model by default. Hook-only controlled functions set /// this to false. - #[serde(default = "default_true", skip_serializing_if = "is_true")] pub expose: bool, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub struct ScenarioRouterV1 { /// Omitted for the deterministic `fixture-model` / `scripted` catalog /// entry used by the integration stack. - #[serde(skip_serializing_if = "Option::is_none")] pub model: Option, pub generations: Vec, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub struct ScenarioGenerationV1 { pub reply: RouterReplyV1, /// Escape hatch for fields whose history is intentionally unstable, such /// as the post-crash request in a recovery reproduction. - #[serde(default, skip_serializing_if = "GenerationMatchOverridesV1::is_empty")] pub match_overrides: GenerationMatchOverridesV1, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub enum RouterReplyV1 { Text { text: String, /// Non-empty chunks produce the complete streaming frame sequence. /// Omitted chunks produce one terminal `done` frame. - #[serde(default, skip_serializing_if = "Vec::is_empty")] chunks: Vec, - #[serde(skip_serializing_if = "Option::is_none")] usage: Option, }, FunctionCall { /// Defaults to `call-`. - #[serde(skip_serializing_if = "Option::is_none")] id: Option, /// Function alias from `functions`. function: String, arguments: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] usage: Option, }, /// Full strict wire contract for cases the typed replies cannot express. @@ -130,32 +107,19 @@ pub enum RouterReplyV1 { }, } -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, Default, PartialEq)] pub struct GenerationMatchOverridesV1 { - #[serde(skip_serializing_if = "Option::is_none")] pub writer_ref: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub request_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub model: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub provider: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub system_prompt: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub messages: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub tools: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub response_format: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub thinking_level: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub max_output_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub provider_options: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub metadata: Option, } @@ -176,8 +140,7 @@ impl GenerationMatchOverridesV1 { } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub struct TriggerBindingSpecV1 { pub trigger: TriggerKindV1, /// Controlled function alias invoked by the trigger. @@ -187,8 +150,7 @@ pub struct TriggerBindingSpecV1 { pub priority: i64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TriggerKindV1 { HookPreTrigger, } @@ -201,21 +163,13 @@ impl TriggerKindV1 { } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct FaultV1 { pub kind: FaultKind, /// Controlled function alias to interrupt. Omitted means the first /// authored function call. - #[serde(skip_serializing_if = "Option::is_none")] pub function: Option, - #[serde(default = "default_one", skip_serializing_if = "is_one")] - #[schemars(range(min = 1))] pub after_target_calls: u64, - #[serde( - default = "default_restart_delay_ms", - skip_serializing_if = "is_default_restart_delay_ms" - )] pub restart_delay_ms: u64, } @@ -273,47 +227,6 @@ impl DeadlinesV1 { } } -#[allow(dead_code)] -#[derive(JsonSchema)] -struct FunctionAliasSchema( - #[schemars(length(min = 1), regex(pattern = "^[A-Za-z0-9_-]+$"))] String, -); - -fn functions_schema(generator: &mut schemars::r#gen::SchemaGenerator) -> schemars::schema::Schema { - let mut schema: schemars::schema::SchemaObject = - BTreeMap::::json_schema(generator).into(); - schema - .object - .as_mut() - .expect("map schema has object validation") - .property_names = Some(Box::new(FunctionAliasSchema::json_schema(generator))); - schema.into() -} - -fn default_true() -> bool { - true -} - -fn is_true(value: &bool) -> bool { - *value -} - -fn default_one() -> u64 { - 1 -} - -fn is_one(value: &u64) -> bool { - *value == default_one() -} - -fn default_restart_delay_ms() -> u64 { - 1_500 -} - -fn is_default_restart_delay_ms(value: &u64) -> bool { - *value == default_restart_delay_ms() -} - fn default_readiness_ms() -> u64 { 60_000 } diff --git a/harness/evals/integration/src/types/scenario/expectations.rs b/harness/evals/integration/src/types/scenario/expectations.rs index bff9cf54f..69b20ec87 100644 --- a/harness/evals/integration/src/types/scenario/expectations.rs +++ b/harness/evals/integration/src/types/scenario/expectations.rs @@ -2,30 +2,24 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::Value; -use super::authored::default_call_id; - /// Typed oracle vocabulary. Common send/completion/lifecycle/router checks -/// have defaults, leaving each fixture to state only scenario-specific facts. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +/// have defaults, leaving each authored scenario to state only +/// scenario-specific facts. +/// +/// The expectation root is authored-only and never serialized. The leaf +/// structs that parameterize compiled invariants +/// ([`MessageCountsExpectationV1`], [`SendFlagsExpectationV1`], +/// [`TerminalExpectationV1`], [`LifecycleExpectationV1`]) keep wire derives. +#[derive(Debug, Clone, PartialEq)] pub struct ExpectationsV1 { - #[serde(skip_serializing_if = "Option::is_none")] pub message_counts: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub assistant_text: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub function_results: Vec, - #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub calls_closed: bool, - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub calls: Vec, - #[serde(default, skip_serializing_if = "SendFlagsExpectationV1::is_default")] pub send_flags: SendFlagsExpectationV1, - #[serde(default = "default_true", skip_serializing_if = "is_true")] pub no_duplicates: bool, - #[serde(default, skip_serializing_if = "TerminalExpectationV1::is_default")] pub terminal: TerminalExpectationV1, - #[serde(default, skip_serializing_if = "LifecycleExpectationV1::is_default")] pub lifecycle: LifecycleExpectationV1, } @@ -59,29 +53,21 @@ pub struct MessageCountsExpectationV1 { pub function_result: u64, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub struct FunctionResultExpectationV1 { - #[serde(default = "default_call_id")] pub function_call_id: String, /// Optional function alias; omitted when any result closing the call is /// acceptable (for example, a synthesized crash-recovery error). - #[serde(skip_serializing_if = "Option::is_none")] pub function: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub content: Option>, - #[serde(skip_serializing_if = "Option::is_none")] pub is_error: Option, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq)] pub struct TargetCallsExpectationV1 { pub function: String, pub count: u64, - #[serde(skip_serializing_if = "Option::is_none")] pub payload: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub payload_subset: Option, } @@ -158,7 +144,3 @@ impl LifecycleExpectationV1 { fn default_true() -> bool { true } - -fn is_true(value: &bool) -> bool { - *value -} diff --git a/harness/evals/integration/tests/scenario_compilation.rs b/harness/evals/integration/tests/scenario_compilation.rs index 5af6b4ddf..f0ae06f45 100644 --- a/harness/evals/integration/tests/scenario_compilation.rs +++ b/harness/evals/integration/tests/scenario_compilation.rs @@ -3,6 +3,7 @@ use std::path::Path; use harness_integration::canonical::canonical_json_pretty; use harness_integration::fixtures::ScenarioFixture; +use harness_integration::scenarios::RegisteredScenario; use harness_integration::types::script::JsonMatcherV1; use serde_json::{json, Value}; @@ -17,20 +18,13 @@ fn snapshot(fixture: &ScenarioFixture) -> Value { }) } -fn scenario_slugs(scenarios: &Path) -> BTreeSet { - std::fs::read_dir(scenarios) - .unwrap_or_else(|error| panic!("reading {}: {error}", scenarios.display())) - .map(|entry| entry.unwrap().path()) - .filter(|path| path.is_dir() && path.join("scenario.yaml").is_file()) - .map(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .unwrap_or_else(|| { - panic!("scenario directory is not valid UTF-8: {}", path.display()) - }) - .to_owned() - }) - .collect() +fn scenarios_root() -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("scenarios") +} + +fn load(entry: &RegisteredScenario) -> ScenarioFixture { + ScenarioFixture::from_registered(entry, &scenarios_root()) + .unwrap_or_else(|error| panic!("compiling {}: {error:#}", entry.slug)) } fn snapshot_slugs(snapshots: &Path) -> BTreeSet { @@ -54,48 +48,53 @@ fn assert_snapshot_inventory(scenarios: &BTreeSet, snapshots: &BTreeSet< let orphaned = snapshots.difference(scenarios).collect::>(); assert!( missing.is_empty() && orphaned.is_empty(), - "compiled snapshot inventory must match scenarios 1:1; \ + "compiled snapshot inventory must match registered scenarios 1:1; \ missing snapshots: {missing:?}; orphaned snapshots: {orphaned:?}" ); } #[test] fn all_compiled_scenarios_match_snapshots() { - let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); - let scenarios = manifest.join("scenarios"); - let snapshots = manifest.join("tests").join("snapshots"); + let snapshots = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("snapshots"); let regenerate = std::env::var_os("REGEN_SCENARIO_SNAPSHOTS").is_some(); - let scenario_slugs = scenario_slugs(&scenarios); + let registered = harness_integration::scenarios::all(); if regenerate { - for slug in &scenario_slugs { - let fixture = ScenarioFixture::load(&scenarios.join(slug)).unwrap(); - let actual = canonical_json_pretty(&snapshot(&fixture)); - let path = snapshots.join(format!("{slug}.compiled.json")); + for entry in ®istered { + let actual = canonical_json_pretty(&snapshot(&load(entry))); + let path = snapshots.join(format!("{}.compiled.json", entry.slug)); std::fs::write(&path, &actual).unwrap(); } } - assert_snapshot_inventory(&scenario_slugs, &snapshot_slugs(&snapshots)); + let registered_slugs: BTreeSet = + registered.iter().map(|entry| entry.slug.clone()).collect(); + assert_snapshot_inventory(®istered_slugs, &snapshot_slugs(&snapshots)); - for slug in &scenario_slugs { - let fixture = ScenarioFixture::load(&scenarios.join(slug)).unwrap(); - let actual = canonical_json_pretty(&snapshot(&fixture)); - let path = snapshots.join(format!("{slug}.compiled.json")); + for entry in ®istered { + let actual = canonical_json_pretty(&snapshot(&load(entry))); + let path = snapshots.join(format!("{}.compiled.json", entry.slug)); let expected = std::fs::read_to_string(&path) .unwrap_or_else(|error| panic!("reading {}: {error}", path.display())); assert_eq!( actual, expected, - "compiled snapshot for {slug}; regenerate with \ - REGEN_SCENARIO_SNAPSHOTS=1 cargo test --test scenario_compilation" + "compiled snapshot for {}; regenerate with \ + REGEN_SCENARIO_SNAPSHOTS=1 cargo test --test scenario_compilation", + entry.slug ); } } #[test] fn inferred_function_history_contains_call_and_result() { - let scenarios = Path::new(env!("CARGO_MANIFEST_DIR")).join("scenarios"); - let fixture = ScenarioFixture::load(&scenarios.join("exactly-once-function")).unwrap(); + let registered = harness_integration::scenarios::all(); + let entry = registered + .iter() + .find(|entry| entry.slug == "exactly-once-function") + .expect("exactly-once-function is registered"); + let fixture = load(entry); let matcher = &fixture.script.generations[1].match_.messages; let JsonMatcherV1::Exact { expected, .. } = matcher else { panic!("function history should use an exact matcher"); diff --git a/harness/evals/integration/tests/schemas.rs b/harness/evals/integration/tests/schemas.rs index 35b0d4f9e..b4b51403f 100644 --- a/harness/evals/integration/tests/schemas.rs +++ b/harness/evals/integration/tests/schemas.rs @@ -10,7 +10,7 @@ use harness_integration::canonical::canonical_json_pretty; use harness_integration::expand::CompiledFixtureV1; use harness_integration::types::recorder::{RecorderEventKind, RecorderEventV1}; use harness_integration::types::scenario::{ - AuthoredScenarioV1, Classification, CompiledScenarioV1, ExecutionReportV1, IntegrationResultV1, + Classification, CompiledScenarioV1, ExecutionReportV1, IntegrationResultV1, }; use harness_integration::types::script::{RouterScriptV1, SchemaVersion1}; @@ -18,9 +18,9 @@ fn goldens() -> Vec<(&'static str, serde_json::Value)> { fn schema() -> serde_json::Value { serde_json::to_value(schemars::schema_for!(T)).expect("schema serializes") } + // The authored layer is code and never serialized, so it has no golden. vec![ ("router-script.v1", schema::()), - ("authored-scenario.v1", schema::()), ("compiled-scenario.v1", schema::()), ("compiled-fixture.v1", schema::()), ("integration-result.v1", schema::()), @@ -62,23 +62,18 @@ fn committed_schemas_match_the_types() { } } -/// Every committed single-file scenario compiles and its authored and strict -/// runtime representations round-trip through their typed mirrors. +/// Every registered scenario compiles and its strict runtime representation +/// round-trips through its typed mirror. The authored layer is code and has +/// no round trip. #[test] -fn committed_scenarios_compile_and_round_trip() { +fn registered_scenarios_compile_and_round_trip() { use harness_integration::fixtures::ScenarioFixture; let scenarios = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("scenarios"); - let mut checked = 0; - for entry in std::fs::read_dir(&scenarios).unwrap() { - let dir = entry.unwrap().path(); - if !dir.join("scenario.yaml").is_file() { - continue; - } - let fixture = ScenarioFixture::load(&dir).unwrap(); - let authored_value = serde_json::to_value(&fixture.authored).unwrap(); - let authored_again: AuthoredScenarioV1 = serde_json::from_value(authored_value).unwrap(); - assert_eq!(fixture.authored, authored_again); + let registered = harness_integration::scenarios::all(); + assert!(!registered.is_empty(), "expected at least one scenario"); + for entry in ®istered { + let fixture = ScenarioFixture::from_registered(entry, &scenarios).unwrap(); let compiled_value = serde_json::to_value(&fixture.scenario).unwrap(); let compiled_again: CompiledScenarioV1 = serde_json::from_value(compiled_value).unwrap(); @@ -92,9 +87,7 @@ fn committed_scenarios_compile_and_round_trip() { let fixture_value = serde_json::to_value(&compiled_fixture).unwrap(); let fixture_again: CompiledFixtureV1 = serde_json::from_value(fixture_value).unwrap(); assert_eq!(compiled_fixture, fixture_again); - checked += 1; } - assert!(checked > 0, "expected at least one committed scenario"); } #[test] @@ -146,12 +139,8 @@ fn compiled_send_is_accepted_by_the_authoritative_harness_contract() { .unwrap(); let validator = jsonschema::JSONSchema::compile(&golden["request_schema"]).unwrap(); let scenarios = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("scenarios"); - for entry in std::fs::read_dir(scenarios).unwrap() { - let dir = entry.unwrap().path(); - if !dir.join("scenario.yaml").is_file() { - continue; - } - let fixture = ScenarioFixture::load(&dir).unwrap(); + for entry in &harness_integration::scenarios::all() { + let fixture = ScenarioFixture::from_registered(entry, &scenarios).unwrap(); let send = serde_json::to_value(&fixture.scenario.send).unwrap(); let errors = validator .validate(&send) @@ -166,39 +155,6 @@ fn compiled_send_is_accepted_by_the_authoritative_harness_contract() { } } -#[test] -fn authored_schema_matches_compiler_safety_constraints() { - use harness_integration::expand::{scenario_template, ScenarioTemplateKind}; - - let schema = serde_json::to_value(schemars::schema_for!(AuthoredScenarioV1)).unwrap(); - let validator = jsonschema::JSONSchema::compile(&schema).unwrap(); - let valid = serde_json::to_value(scenario_template( - "C-E2E-SCHEMA", - "Validate authored schema constraints.", - ScenarioTemplateKind::Crash, - )) - .unwrap(); - assert!(validator.is_valid(&valid)); - - let mut unsafe_id = valid.clone(); - unsafe_id["id"] = serde_json::json!("../../escape"); - assert!(!validator.is_valid(&unsafe_id)); - - let mut unsafe_alias = valid.clone(); - let functions = unsafe_alias["functions"].as_object_mut().unwrap(); - let function = functions.remove("record").unwrap(); - functions.insert("../record".to_string(), function); - assert!(!validator.is_valid(&unsafe_alias)); - - let mut zero_timeout = valid.clone(); - zero_timeout["timeouts"]["teardown_ms"] = serde_json::json!(0); - assert!(!validator.is_valid(&zero_timeout)); - - let mut zero_fault_threshold = valid; - zero_fault_threshold["fault"]["after_target_calls"] = serde_json::json!(0); - assert!(!validator.is_valid(&zero_fault_threshold)); -} - #[test] fn compiled_schema_matches_runtime_safety_constraints() { use harness_integration::expand::{compile_scenario, scenario_template, ScenarioTemplateKind}; From 4d935b5fc5689aa046167dffe15ab5ab3638f736 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Tue, 21 Jul 2026 10:01:06 -0300 Subject: [PATCH 02/11] (MOT-4107) refactor(harness): add recorder, recovery, and release scenario builders Scenario modules repeated the recorder fixture, the recovery-boundary match-override block, and reached into raw ReleaseActionV1. Lift all three into the builder vocabulary: - Function::recorder() for the canonical string-in/recorded-out fixture - Reply::...().recovery_boundary() for durable-outcome-only grading at fault restarts and hook releases - Release::execute() / Release::deliver() for the held-call action Compiled snapshots are byte-identical; no behavior change. --- harness/evals/integration/README.md | 15 ++-- .../integration/src/scenarios/builder.rs | 68 ++++++++++++++++++- .../src/scenarios/crash_recovery_507.rs | 31 +-------- .../src/scenarios/exactly_once_function.rs | 17 +---- .../src/scenarios/hold_mutation_505.rs | 33 +-------- .../src/scenarios/hook_held_release_506.rs | 33 +-------- 6 files changed, 86 insertions(+), 111 deletions(-) diff --git a/harness/evals/integration/README.md b/harness/evals/integration/README.md index 02158ee6c..aa4fdfb0e 100644 --- a/harness/evals/integration/README.md +++ b/harness/evals/integration/README.md @@ -103,11 +103,16 @@ excluded from `run --scenario all`. `render` prints deterministic canonical JSON with the complete compiled request, router script, expectations, and system prompt. -Function aliases become `::`. Chain `.hidden()` for hook-only -functions. `Send::message(...).allow([...])` can narrow the exposed aliases -or be an empty list to disable dispatch. Typed text and function-call replies -cover normal cases; `.match_overrides(...)` and `RouterReplyV1::Raw` remain -escape hatches for recovery boundaries and unusual wire contracts. +`Function::recorder()` is the canonical string-in/`recorded`-out fixture; +`Function::new(...)` builds any other controlled function and `.hidden()` +marks a hook-only one. Function aliases become `::`. +`Send::message(...).allow([...])` can narrow the exposed aliases or be an +empty list to disable dispatch. `Release::execute()` and `Release::deliver()` +name the held-call action. Typed text and function-call replies cover normal +cases; `.recovery_boundary()` grades a reply against the durable outcome only, +where a fault restart or hook release may rebuild the request. `.match_overrides(...)` +and `RouterReplyV1::Raw` remain the deeper escape hatches for unusual wire +contracts. Timeout defaults are 60 seconds for readiness, 60 seconds for the scenario, and 15 seconds for teardown. Positive values can be overridden with the diff --git a/harness/evals/integration/src/scenarios/builder.rs b/harness/evals/integration/src/scenarios/builder.rs index 8bdb51d56..5ac0f879e 100644 --- a/harness/evals/integration/src/scenarios/builder.rs +++ b/harness/evals/integration/src/scenarios/builder.rs @@ -10,7 +10,7 @@ //! chunks emits one terminal `done` frame, faults interrupt the first target //! call after 1500 ms, and releases target `call-1`. -use serde_json::Value; +use serde_json::{json, Value}; use crate::types::frames::Usage; use crate::types::scenario::{ @@ -157,6 +157,25 @@ impl ScenarioFunctionV1 { } } + /// The canonical recorder fixture: one required string `value`, returning + /// a durable `recorded` text result. It is the most common controlled + /// function; chain [`Self::hidden`] for a hook-only variant. + pub fn recorder() -> Self { + Self::new( + "Record one integration fixture value.", + json!({ + "type": "object", + "additionalProperties": false, + "properties": { "value": { "type": "string" } }, + "required": ["value"] + }), + json!({ + "content": [{ "type": "text", "text": "recorded" }], + "is_error": false + }), + ) + } + /// Hook-only controlled functions are never exposed to the model. pub fn hidden(mut self) -> Self { self.expose = false; @@ -209,6 +228,14 @@ impl TextReply { pub fn match_overrides(self, overrides: GenerationMatchOverridesV1) -> ScenarioGenerationV1 { with_overrides(self.into(), overrides) } + + /// Grade this reply against the durable outcome only, at a recovery + /// boundary (fault restart or hook release) where the engine may rebuild + /// the request differently. Shorthand for [`Self::match_overrides`] with + /// the recovery policy. + pub fn recovery_boundary(self) -> ScenarioGenerationV1 { + with_overrides(self.into(), recovery_overrides()) + } } impl From for ScenarioGenerationV1 { @@ -236,6 +263,14 @@ impl FunctionCallReply { pub fn match_overrides(self, overrides: GenerationMatchOverridesV1) -> ScenarioGenerationV1 { with_overrides(self.into(), overrides) } + + /// Grade this reply against the durable outcome only, at a recovery + /// boundary (fault restart or hook release) where the engine may rebuild + /// the request differently. Shorthand for [`Self::match_overrides`] with + /// the recovery policy. + pub fn recovery_boundary(self) -> ScenarioGenerationV1 { + with_overrides(self.into(), recovery_overrides()) + } } impl From for ScenarioGenerationV1 { @@ -264,6 +299,20 @@ fn with_overrides( generation } +/// The loose match a recovery-boundary reply needs. After a fault restart or a +/// hook release the engine may legitimately reconstruct the request +/// differently, so grade the durable shape and leave the reconstructed request +/// id and body free. +fn recovery_overrides() -> GenerationMatchOverridesV1 { + GenerationMatchOverridesV1 { + request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), + system_prompt: Some(present()), + messages: Some(present()), + tools: Some(present()), + ..Default::default() + } +} + fn input_output_usage(input: u64, output: u64) -> Usage { Usage { input: Some(input), @@ -325,6 +374,23 @@ impl FaultV1 { } } +/// Entry point for [`ReleaseActionV1`], the action applied to the held +/// deterministic first call by [`AuthoredScenarioV1::release`]. +pub struct Release; + +impl Release { + /// Release the held call for execution against its target function. + pub fn execute() -> ReleaseActionV1 { + ReleaseActionV1::Execute + } + + /// Release the held call by delivering its recorded result without + /// re-execution. + pub fn deliver() -> ReleaseActionV1 { + ReleaseActionV1::Deliver + } +} + /// Authoring name for [`ExpectationsV1`]. pub type Expect = ExpectationsV1; diff --git a/harness/evals/integration/src/scenarios/crash_recovery_507.rs b/harness/evals/integration/src/scenarios/crash_recovery_507.rs index 9020be60e..0c1a5689a 100644 --- a/harness/evals/integration/src/scenarios/crash_recovery_507.rs +++ b/harness/evals/integration/src/scenarios/crash_recovery_507.rs @@ -4,8 +4,6 @@ use serde_json::json; -use crate::types::scenario::GenerationMatchOverridesV1; - use super::builder::*; pub(super) fn scenario() -> AuthoredScenario { @@ -15,36 +13,11 @@ pub(super) fn scenario() -> AuthoredScenario { ) .quarantine() .send(Send::message("Call the recorder once.")) - .function( - "record", - Function::new( - "Record one integration fixture value.", - json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }), - json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - ), - ) + .function("record", Function::recorder()) .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) // Recovery can legitimately reconstruct the second request differently; // this reproduction grades the durable outcome instead. - .generation( - Reply::text("recovered") - .usage(20, 2) - .match_overrides(GenerationMatchOverridesV1 { - request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), - system_prompt: Some(present()), - messages: Some(present()), - tools: Some(present()), - ..Default::default() - }), - ) + .generation(Reply::text("recovered").usage(20, 2).recovery_boundary()) .fault(Fault::engine_sigkill()) .scenario_timeout_ms(120_000) .expect( diff --git a/harness/evals/integration/src/scenarios/exactly_once_function.rs b/harness/evals/integration/src/scenarios/exactly_once_function.rs index 2646f7601..2bab9daf7 100644 --- a/harness/evals/integration/src/scenarios/exactly_once_function.rs +++ b/harness/evals/integration/src/scenarios/exactly_once_function.rs @@ -10,22 +10,7 @@ pub(super) fn scenario() -> AuthoredScenario { "An allow-listed native function executes exactly once with a durable result.", ) .send(Send::message("Call the recorder once.")) - .function( - "record", - Function::new( - "Record one integration fixture value.", - json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }), - json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - ), - ) + .function("record", Function::recorder()) .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) .generation(Reply::text("recorded once").usage(18, 2)) .expect( diff --git a/harness/evals/integration/src/scenarios/hold_mutation_505.rs b/harness/evals/integration/src/scenarios/hold_mutation_505.rs index fd97b6b2a..78975ed74 100644 --- a/harness/evals/integration/src/scenarios/hold_mutation_505.rs +++ b/harness/evals/integration/src/scenarios/hold_mutation_505.rs @@ -4,8 +4,6 @@ use serde_json::json; -use crate::types::scenario::GenerationMatchOverridesV1; - use super::builder::*; pub(super) fn scenario() -> AuthoredScenario { @@ -15,22 +13,7 @@ pub(super) fn scenario() -> AuthoredScenario { ) .quarantine() .send(Send::message("Call the recorder once.")) - .function( - "record", - Function::new( - "Record one integration fixture value.", - json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }), - json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - ), - ) + .function("record", Function::recorder()) .function( "hook-gate", Function::new( @@ -44,19 +27,9 @@ pub(super) fn scenario() -> AuthoredScenario { .hidden(), ) .binding(Binding::hook_pre_trigger("hook-gate", ["record"], 10)) - .release(crate::types::scenario::ReleaseActionV1::Execute) + .release(Release::execute()) .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) - .generation( - Reply::text("approved and recorded") - .usage(20, 3) - .match_overrides(GenerationMatchOverridesV1 { - request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), - system_prompt: Some(present()), - messages: Some(present()), - tools: Some(present()), - ..Default::default() - }), - ) + .generation(Reply::text("approved and recorded").usage(20, 3).recovery_boundary()) .expect( Expect::new() .calls_closed() diff --git a/harness/evals/integration/src/scenarios/hook_held_release_506.rs b/harness/evals/integration/src/scenarios/hook_held_release_506.rs index eb180c33a..2ba4c8047 100644 --- a/harness/evals/integration/src/scenarios/hook_held_release_506.rs +++ b/harness/evals/integration/src/scenarios/hook_held_release_506.rs @@ -4,8 +4,6 @@ use serde_json::json; -use crate::types::scenario::GenerationMatchOverridesV1; - use super::builder::*; pub(super) fn scenario() -> AuthoredScenario { @@ -15,22 +13,7 @@ pub(super) fn scenario() -> AuthoredScenario { ) .quarantine() .send(Send::message("Call the recorder once.")) - .function( - "record", - Function::new( - "Record one integration fixture value.", - json!({ - "type": "object", - "additionalProperties": false, - "properties": { "value": { "type": "string" } }, - "required": ["value"] - }), - json!({ - "content": [{ "type": "text", "text": "recorded" }], - "is_error": false - }), - ), - ) + .function("record", Function::recorder()) .function( "hook-mutate", Function::new( @@ -54,19 +37,9 @@ pub(super) fn scenario() -> AuthoredScenario { ) .binding(Binding::hook_pre_trigger("hook-mutate", ["record"], 10)) .binding(Binding::hook_pre_trigger("hook-hold", ["record"], 20)) - .release(crate::types::scenario::ReleaseActionV1::Execute) + .release(Release::execute()) .generation(Reply::function_call("record", json!({ "value": "expected" })).usage(8, 4)) - .generation( - Reply::text("released and recorded") - .usage(20, 3) - .match_overrides(GenerationMatchOverridesV1 { - request_id: Some(regex("^t_[0-9a-f]{32}:[0-9]+$")), - system_prompt: Some(present()), - messages: Some(present()), - tools: Some(present()), - ..Default::default() - }), - ) + .generation(Reply::text("released and recorded").usage(20, 3).recovery_boundary()) .expect( Expect::new() .calls_closed() From 0e520e5d1c3cd94a4a951543cc773ef614af8b5c Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Tue, 21 Jul 2026 14:12:10 -0300 Subject: [PATCH 03/11] (MOT-4107) test(console): add deterministic Playwright E2E --- console/web/e2e/durable-hydration.spec.ts | 44 ++ console/web/e2e/exactly-once-function.spec.ts | 34 + console/web/e2e/harness-stack.ts | 314 ++++++++ console/web/e2e/ui-send.spec.ts | 33 + console/web/package.json | 6 +- console/web/playwright.config.ts | 23 + console/web/pnpm-lock.yaml | 38 + console/web/pnpm-workspace.yaml | 2 + .../web/src/components/chat/LexicalShell.tsx | 1 + console/web/src/components/chat/Message.tsx | 4 +- .../function-call/FunctionCallCard.tsx | 7 +- .../web/src/hooks/use-conversations.test.ts | 20 + console/web/src/hooks/use-conversations.ts | 36 +- console/web/tsconfig.e2e.json | 8 + console/web/vite.config.ts | 5 +- harness/evals/integration/Cargo.lock | 11 + harness/evals/integration/Cargo.toml | 2 +- .../evals/integration/src/expand/render.rs | 1 - .../evals/integration/src/fixtures/loading.rs | 6 +- .../evals/integration/src/fixtures/tests.rs | 9 +- harness/evals/integration/src/main.rs | 162 ++++- .../evals/integration/src/recorder/service.rs | 39 +- harness/evals/integration/src/scenario.rs | 2 + .../evals/integration/src/scenario/serve.rs | 674 ++++++++++++++++++ .../integration/src/scenarios/builder.rs | 7 + .../src/scenarios/console_streamed_text.rs | 33 + .../src/scenarios/hold_mutation_505.rs | 22 +- .../src/scenarios/hook_held_release_506.rs | 36 +- .../evals/integration/src/scenarios/mod.rs | 19 + harness/evals/integration/src/stack/bins.rs | 3 + .../evals/integration/src/stack/manifest.rs | 8 +- .../evals/integration/src/stack/supervisor.rs | 15 + harness/evals/integration/src/stack/tests.rs | 3 + .../console-streamed-text.compiled.json | 287 ++++++++ 34 files changed, 1837 insertions(+), 77 deletions(-) create mode 100644 console/web/e2e/durable-hydration.spec.ts create mode 100644 console/web/e2e/exactly-once-function.spec.ts create mode 100644 console/web/e2e/harness-stack.ts create mode 100644 console/web/e2e/ui-send.spec.ts create mode 100644 console/web/playwright.config.ts create mode 100644 console/web/pnpm-workspace.yaml create mode 100644 console/web/tsconfig.e2e.json create mode 100644 harness/evals/integration/src/scenario/serve.rs create mode 100644 harness/evals/integration/src/scenarios/console_streamed_text.rs create mode 100644 harness/evals/integration/tests/snapshots/console-streamed-text.compiled.json diff --git a/console/web/e2e/durable-hydration.spec.ts b/console/web/e2e/durable-hydration.spec.ts new file mode 100644 index 000000000..126cb083c --- /dev/null +++ b/console/web/e2e/durable-hydration.spec.ts @@ -0,0 +1,44 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +test.use({ scenario: 'streamed-text' }) + +test('hydrates a durable transcript again after a page reload', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await stack.trigger('harness::send', stack.ready.send) + expect(await completed).toMatchObject({ status: 'completed' }) + + await openSession(page, stack.ready) + await expect( + page.locator('[data-message-role="user"]', { + hasText: stack.ready.message, + }), + ).toHaveCount(1) + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'fixture complete', + }), + ).toHaveCount(1) + + await page.reload() + await page + .getByRole('button', { + name: `open ${stack.ready.session.title}`, + exact: true, + }) + .click() + await expect( + page.locator('[data-message-role="user"]', { + hasText: stack.ready.message, + }), + ).toHaveCount(1) + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'fixture complete', + }), + ).toHaveCount(1) + + expectPassingResult(await stack.finish()) +}) diff --git a/console/web/e2e/exactly-once-function.spec.ts b/console/web/e2e/exactly-once-function.spec.ts new file mode 100644 index 000000000..4599bbfe6 --- /dev/null +++ b/console/web/e2e/exactly-once-function.spec.ts @@ -0,0 +1,34 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +test.use({ scenario: 'exactly-once-function' }) + +test('renders one completed function call and its durable result', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await stack.trigger('harness::send', stack.ready.send) + expect(await completed).toMatchObject({ status: 'completed' }) + + await openSession(page, stack.ready) + const functionId = stack.ready.functions.record + expect(functionId).toBeTruthy() + const card = page.locator('[data-message-role="function-call"]', { + hasText: functionId, + }) + await expect(card).toHaveCount(1) + await expect(card).toHaveAttribute('data-function-id', functionId) + await expect(card).toHaveAttribute('data-function-status', 'done') + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'recorded once', + }), + ).toHaveCount(1) + + const result = await stack.finish() + expectPassingResult(result) + expect( + result.invariants.find((invariant) => invariant.id === 'target.calls') + ?.actual, + ).toMatchObject({ count: 1 }) +}) diff --git a/console/web/e2e/harness-stack.ts b/console/web/e2e/harness-stack.ts new file mode 100644 index 000000000..0a8dca3f2 --- /dev/null +++ b/console/web/e2e/harness-stack.ts @@ -0,0 +1,314 @@ +import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, watch } from 'node:fs/promises' +import path from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import type { Page } from '@playwright/test' +import { test as base, expect } from '@playwright/test' +import { type ISdk, registerWorker } from 'iii-browser-sdk' + +interface ReadyManifest { + schema_version: '1' + run_id: string + scenario_id: string + scenario_slug: string + driver: 'direct' | 'console' + run_root: string + result_path: string + console_url: string + engine_url: string + session: { id: string; title: string } + model: { id: string; provider: string } + message: string + functions: Record + send?: Record +} + +interface InvariantResult { + id: string + passed: boolean + expected: unknown + actual: unknown + evidence_refs: string[] +} + +export interface ServeResult { + schema_version: '1' + scenario_id: string + classification: + | 'pass' + | 'setup_error' + | 'contract_failure' + | 'timeout' + | 'process_crash' + | 'runner_error' + invariants: InvariantResult[] + skipped_invariants: string[] + artifacts: string[] +} + +interface TurnCompletedEvent { + session_id: string + turn_id: string + status: 'completed' | 'cancelled' | 'failed' + timestamp: number +} + +export interface HarnessStack { + ready: ReadyManifest + trigger(functionId: string, payload: unknown): Promise + waitForTurnCompleted(): Promise + finish(): Promise +} + +interface FixtureOptions { + scenario: string +} + +interface FixtureValues { + stack: HarnessStack +} + +function required(name: string): string { + const value = process.env[name] + if (!value) throw new Error(`${name} is required for Console E2E`) + return value +} + +function workerArgs(): string[] { + return [ + ['queue', 'QUEUE_BIN'], + ['iii-directory', 'III_DIRECTORY_BIN'], + ['session-manager', 'SESSION_MANAGER_BIN'], + ['context-manager', 'CONTEXT_MANAGER_BIN'], + ].flatMap(([name, env]) => ['--worker-bin', `${name}=${required(env)}`]) +} + +async function waitForReady( + readyFile: string, + childExit: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, +): Promise { + const read = async (): Promise => { + try { + return JSON.parse(await readFile(readyFile, 'utf8')) as ReadyManifest + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ENOENT' || error instanceof SyntaxError) return null + throw error + } + } + const existing = await read() + if (existing) return existing + + const parent = path.dirname(readyFile) + const expectedName = path.basename(readyFile) + const changes = watch(parent) + const timeout = delay(70_000).then(() => { + throw new Error(`timed out waiting for ${readyFile}`) + }) + const exited = childExit.then(({ code, signal }) => { + throw new Error( + `harness-integration exited before ready (code=${String(code)}, signal=${String(signal)})`, + ) + }) + const appeared = (async () => { + for await (const event of changes) { + if (event.filename && event.filename !== expectedName) continue + const manifest = await read() + if (manifest) return manifest + } + throw new Error(`ready-file watcher closed before ${readyFile} appeared`) + })() + try { + return await Promise.race([appeared, exited, timeout]) + } finally { + await changes.return?.() + } +} + +function childExit( + child: ChildProcessWithoutNullStreams, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + return new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })) + }) +} + +function armCompletion( + sdk: ISdk, + ready: ReadyManifest, +): Promise { + const functionId = `console-e2e::turn-completed::${ready.run_id}` + let functionRef: ReturnType | undefined + let triggerRef: ReturnType | undefined + let timer: NodeJS.Timeout | undefined + const cleanup = () => { + if (timer) clearTimeout(timer) + try { + triggerRef?.unregister() + } catch { + // The stack may already be shutting down. + } + try { + functionRef?.unregister() + } catch { + // The stack may already be shutting down. + } + } + const completed = new Promise((resolve, reject) => { + functionRef = sdk.registerFunction( + functionId, + async (payload) => { + const event = payload as TurnCompletedEvent + if (event.session_id !== ready.session.id) return null + cleanup() + resolve(event) + return null + }, + { metadata: { internal: true } }, + ) + triggerRef = sdk.registerTrigger({ + type: 'harness::turn-completed', + function_id: functionId, + config: { session_id: ready.session.id }, + }) + timer = setTimeout(() => { + cleanup() + reject(new Error('harness::turn-completed was not delivered')) + }, 60_000) + }) + return completed +} + +export const test = base.extend({ + scenario: ['', { scope: 'worker', option: true }], + stack: async ({ scenario }, use, testInfo) => { + if (!scenario) throw new Error('test.use({ scenario }) is required') + const artifactsRoot = path.resolve( + process.env.CONSOLE_E2E_ARTIFACTS_DIR ?? + path.join(testInfo.project.outputDir, '..'), + ) + await mkdir(artifactsRoot, { recursive: true }) + const controlDir = await mkdtemp(path.join(artifactsRoot, 'runner-')) + const readyFile = path.join(controlDir, 'ready.json') + const args = [ + 'serve', + '--scenario', + scenario, + '--engine-bin', + required('III_BIN'), + '--harness-bin', + required('HARNESS_BIN'), + '--console-bin', + required('CONSOLE_BIN'), + '--artifacts-dir', + artifactsRoot, + '--ready-file', + readyFile, + ...workerArgs(), + ] + const child = spawn(required('HARNESS_INTEGRATION_BIN'), args, { + stdio: ['pipe', 'pipe', 'pipe'], + }) + const exit = childExit(child) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)) + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)) + + let sdk: ISdk | undefined + let finalized: Promise | undefined + const finish = (): Promise => { + if (finalized) return finalized + finalized = (async () => { + if (sdk) await sdk.shutdown().catch(() => undefined) + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGTERM') + } + let exited = await Promise.race([exit, delay(30_000).then(() => null)]) + if (!exited) { + child.kill('SIGKILL') + exited = await exit + } + await testInfo.attach('harness-integration.stdout', { + body: Buffer.concat(stdout), + contentType: 'text/plain', + }) + await testInfo.attach('harness-integration.stderr', { + body: Buffer.concat(stderr), + contentType: 'text/plain', + }) + const result = JSON.parse( + await readFile(ready.result_path, 'utf8'), + ) as ServeResult + await testInfo.attach('serve-result', { + body: JSON.stringify(result, null, 2), + contentType: 'application/json', + }) + return result + })() + return finalized + } + + let ready!: ReadyManifest + try { + ready = await waitForReady(readyFile, exit) + } catch (error) { + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGTERM') + } + await exit.catch(() => undefined) + await testInfo.attach('harness-integration.stdout', { + body: Buffer.concat(stdout), + contentType: 'text/plain', + }) + await testInfo.attach('harness-integration.stderr', { + body: Buffer.concat(stderr), + contentType: 'text/plain', + }) + throw error + } + const connectedSdk = registerWorker(ready.engine_url) + sdk = connectedSdk + const stack: HarnessStack = { + ready, + trigger: (functionId: string, payload: unknown) => + connectedSdk.trigger({ + function_id: functionId, + payload, + timeoutMs: 30_000, + }), + waitForTurnCompleted: () => armCompletion(connectedSdk, ready), + finish, + } + + try { + await use(stack) + } finally { + if (!finalized) { + await finish().catch(() => undefined) + } + await rm(controlDir, { recursive: true, force: true }) + } + }, +}) + +export { expect } + +export async function openSession( + page: Page, + ready: ReadyManifest, +): Promise { + await page.goto(ready.console_url) + const session = page.getByRole('button', { + name: `open ${ready.session.title}`, + exact: true, + }) + await session.click() + await expect(session).toHaveAttribute('aria-current', 'page') +} + +export function expectPassingResult(result: ServeResult): void { + expect(result.classification).toBe('pass') + expect(result.skipped_invariants).toEqual(['send.flags']) + expect(result.invariants.every((invariant) => invariant.passed)).toBe(true) +} diff --git a/console/web/e2e/ui-send.spec.ts b/console/web/e2e/ui-send.spec.ts new file mode 100644 index 000000000..5c1405dc4 --- /dev/null +++ b/console/web/e2e/ui-send.spec.ts @@ -0,0 +1,33 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +test.use({ scenario: 'console-streamed-text' }) + +test('sends from the production Console and renders streamed text', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await openSession(page, stack.ready) + + await page + .getByRole('textbox', { name: 'message composer' }) + .fill(stack.ready.message) + await page.getByRole('button', { name: 'send message' }).click() + + await expect( + page.locator('[data-message-role="user"]', { + hasText: stack.ready.message, + }), + ).toHaveCount(1) + expect(await completed).toMatchObject({ + session_id: stack.ready.session.id, + status: 'completed', + }) + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'console fixture complete', + }), + ).toHaveCount(1) + + expectPassingResult(await stack.finish()) +}) diff --git a/console/web/package.json b/console/web/package.json index bf7cbdd6b..2466dce68 100644 --- a/console/web/package.json +++ b/console/web/package.json @@ -3,16 +3,19 @@ "private": true, "version": "0.0.0", "type": "module", - "packageManager": "pnpm@10.18.2", + "packageManager": "pnpm@11.13.1", "scripts": { "dev": "vite", "build": "tsc -b && vite build", "preview": "vite preview", "typecheck": "tsc -b --noEmit", + "typecheck:e2e": "tsc -p tsconfig.e2e.json", "lint": "biome check .", "lint:fix": "biome check --write .", "test": "vitest run", "test:watch": "vitest", + "test:e2e": "playwright test", + "test:e2e:install": "playwright install chromium", "storybook": "storybook dev -p 6006", "build-storybook": "storybook build" }, @@ -45,6 +48,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", + "@playwright/test": "^1.61.1", "@storybook/addon-a11y": "^10.4.1", "@storybook/addon-docs": "^10.4.1", "@storybook/react-vite": "^10.4.1", diff --git a/console/web/playwright.config.ts b/console/web/playwright.config.ts new file mode 100644 index 000000000..7f5c7775d --- /dev/null +++ b/console/web/playwright.config.ts @@ -0,0 +1,23 @@ +import path from 'node:path' +import { defineConfig } from '@playwright/test' + +const artifactsRoot = + process.env.CONSOLE_E2E_ARTIFACTS_DIR ?? + path.resolve(import.meta.dirname, '../../target/console-e2e') + +export default defineConfig({ + testDir: './e2e', + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 120_000, + expect: { timeout: 15_000 }, + reporter: [['list']], + outputDir: path.join(artifactsRoot, 'playwright-output'), + use: { + browserName: 'chromium', + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, +}) diff --git a/console/web/pnpm-lock.yaml b/console/web/pnpm-lock.yaml index 061299d48..2d13033ca 100644 --- a/console/web/pnpm-lock.yaml +++ b/console/web/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: '@biomejs/biome': specifier: ^2.4.15 version: 2.4.15 + '@playwright/test': + specifier: ^1.61.1 + version: 1.61.1 '@storybook/addon-a11y': specifier: ^10.4.1 version: 10.4.1(storybook@10.4.1(@testing-library/dom@10.4.1)(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)) @@ -826,6 +829,11 @@ packages: resolution: {integrity: sha512-sWHv11TMoqKxKDgTIk5VbhQjdPhs8DCcBxbjh3mRlS3YOM/OcrWoGX6MM8eBGn9cUu3M46Py0JnxsG2nJaFTuA==} engines: {vscode: ^1.0.0} + '@playwright/test@1.61.1': + resolution: {integrity: sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==} + engines: {node: '>=18'} + hasBin: true + '@preact/signals-core@1.14.2': resolution: {integrity: sha512-RZHdBj9ZF4n40Rp4jS052EHHjBWf96P9oNdXPfhQTovCuWY9iQn3Gq+gOTJSgBO9A/JBuPfMOWsSX/lIU9Pc/A==} @@ -2124,6 +2132,11 @@ packages: picomatch: optional: true + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -2568,6 +2581,16 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} + playwright-core@1.61.1: + resolution: {integrity: sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.61.1: + resolution: {integrity: sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==} + engines: {node: '>=18'} + hasBin: true + postcss@8.5.14: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} @@ -3648,6 +3671,10 @@ snapshots: '@pierre/theme@1.0.3': {} + '@playwright/test@1.61.1': + dependencies: + playwright: 1.61.1 + '@preact/signals-core@1.14.2': {} '@radix-ui/number@1.1.1': {} @@ -4858,6 +4885,9 @@ snapshots: optionalDependencies: picomatch: 4.0.4 + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -5521,6 +5551,14 @@ snapshots: picomatch@4.0.4: {} + playwright-core@1.61.1: {} + + playwright@1.61.1: + dependencies: + playwright-core: 1.61.1 + optionalDependencies: + fsevents: 2.3.2 + postcss@8.5.14: dependencies: nanoid: 3.3.12 diff --git a/console/web/pnpm-workspace.yaml b/console/web/pnpm-workspace.yaml new file mode 100644 index 000000000..5ed0b5af0 --- /dev/null +++ b/console/web/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + esbuild: true diff --git a/console/web/src/components/chat/LexicalShell.tsx b/console/web/src/components/chat/LexicalShell.tsx index 76082da84..fe4c29fa8 100644 --- a/console/web/src/components/chat/LexicalShell.tsx +++ b/console/web/src/components/chat/LexicalShell.tsx @@ -252,6 +252,7 @@ export function LexicalShell({ diff --git a/console/web/src/components/chat/Message.tsx b/console/web/src/components/chat/Message.tsx index bdf822b13..9dc4901b5 100644 --- a/console/web/src/components/chat/Message.tsx +++ b/console/web/src/components/chat/Message.tsx @@ -266,7 +266,7 @@ function SpawnTaskMessage({ message }: { message: UserMessageType }) { function UserMessage({ message }: { message: UserMessageType }) { return ( -
+
you
@@ -292,7 +292,7 @@ function UserMessage({ message }: { message: UserMessageType }) { function AssistantMessage({ message }: { message: AssistantMessageType }) { const showCaret = !!message.streaming return ( -
+
assistant {message.model ? ( diff --git a/console/web/src/components/function-call/FunctionCallCard.tsx b/console/web/src/components/function-call/FunctionCallCard.tsx index e19c98eaf..a66445d20 100644 --- a/console/web/src/components/function-call/FunctionCallCard.tsx +++ b/console/web/src/components/function-call/FunctionCallCard.tsx @@ -10,11 +10,11 @@ import { DirectoryToolView, } from '@/components/chat/directory' import { EngineFunctionIdLabel, EngineToolView } from '@/components/chat/engine' +import { FpFunctionIdLabel, FpToolView } from '@/components/chat/fp' import { HarnessFunctionIdLabel, HarnessToolView, } from '@/components/chat/harness' -import { FpFunctionIdLabel, FpToolView } from '@/components/chat/fp' import { RouterFunctionIdLabel, RouterToolView } from '@/components/chat/router' import { SandboxFunctionIdLabel, @@ -351,6 +351,11 @@ export function FunctionCallCard({ !embedded && 'border border-rule bg-bg', )} data-message-id={message.id} + data-message-role="function-call" + data-function-id={message.functionId} + data-function-status={ + pending ? 'pending' : running ? 'running' : errored ? 'error' : 'done' + } >