diff --git a/.github/benchmark-site/execution-transcript.js b/.github/benchmark-site/execution-transcript.js index a2eff130d..06215204c 100644 --- a/.github/benchmark-site/execution-transcript.js +++ b/.github/benchmark-site/execution-transcript.js @@ -27,6 +27,28 @@ return String(entry?.entry_id || fallback).replace(/[^a-zA-Z0-9_-]/g, "-"); } + function displayFunctionId(functionId, argumentsValue) { + const wrappedFunction = + functionId === "agent_trigger" && + argumentsValue && + typeof argumentsValue === "object" && + typeof argumentsValue.function === "string" + ? argumentsValue.function.trim() + : ""; + return wrappedFunction || functionId || "unknown function"; + } + + function displayFunctionArguments(functionId, argumentsValue) { + if ( + functionId === "agent_trigger" && + argumentsValue && + typeof argumentsValue === "object" + ) { + return argumentsValue.payload ?? null; + } + return argumentsValue ?? null; + } + function normalizeTranscript(messages) { const events = []; const calls = new Map(); @@ -61,8 +83,8 @@ ), kind: "tool", callId, - functionId: block.function_id || "unknown function", - arguments: block.arguments ?? null, + functionId: displayFunctionId(block.function_id, block.arguments), + arguments: displayFunctionArguments(block.function_id, block.arguments), timestamp: message.timestamp || null, result: null, isError: false, @@ -160,6 +182,8 @@ return { contentBlocks, + displayFunctionArguments, + displayFunctionId, filterTranscript, messageText, normalizeTranscript, diff --git a/.github/benchmark-site/execution-transcript.test.cjs b/.github/benchmark-site/execution-transcript.test.cjs index d162d06ea..d4aa55c30 100644 --- a/.github/benchmark-site/execution-transcript.test.cjs +++ b/.github/benchmark-site/execution-transcript.test.cjs @@ -2,12 +2,34 @@ const test = require("node:test"); const assert = require("node:assert/strict"); const { + displayFunctionArguments, + displayFunctionId, filterTranscript, messageText, normalizeTranscript, summarizeTranscript, } = require("./execution-transcript.js"); +test("shows the wrapped function name instead of agent_trigger", () => { + const wrappedArguments = { + function: "database::query", + payload: { db: "primary", sql: "SELECT 1" }, + }; + assert.equal( + displayFunctionId("agent_trigger", wrappedArguments), + "database::query", + ); + assert.deepEqual(displayFunctionArguments("agent_trigger", wrappedArguments), { + db: "primary", + sql: "SELECT 1", + }); + assert.deepEqual(displayFunctionArguments("state::get", { scope: "test" }), { + scope: "test", + }); + assert.equal(displayFunctionId("state::get", { scope: "test" }), "state::get"); + assert.equal(displayFunctionId("agent_trigger", { payload: {} }), "agent_trigger"); +}); + test("extracts readable text from current and legacy message content", () => { assert.equal(messageText(" Legacy response. "), " Legacy response. "); assert.equal( @@ -39,8 +61,11 @@ test("pairs function results with calls and marks recovered errors", () => { { type: "function_call", id: "call-ok", - function_id: "state::get", - arguments: { key: "status" }, + function_id: "agent_trigger", + arguments: { + function: "state::get", + payload: { key: "status" }, + }, }, { type: "function_call", @@ -85,6 +110,7 @@ test("pairs function results with calls and marks recovered errors", () => { ], ); assert.equal(events[2].result.details.value, "ready"); + assert.deepEqual(events[2].arguments, { key: "status" }); assert.equal(events[2].status, "completed"); assert.equal(events[3].isError, true); assert.equal(events[3].result.details.error, "command failed"); diff --git a/.github/workflows/_harness-e2e.yml b/.github/workflows/_harness-e2e.yml index d0ce4c27f..18750200a 100644 --- a/.github/workflows/_harness-e2e.yml +++ b/.github/workflows/_harness-e2e.yml @@ -14,10 +14,15 @@ on: type: number default: 2 quality_advisory: - description: Report score failures without failing the workflow + description: Only fail degraded results below the CI score floor required: false type: boolean default: false + ci_score_floor: + description: Minimum median score allowed by the advisory CI policy + required: false + type: number + default: 50 subjects: description: JSON array of subject entries with id, model, and provider required: false @@ -301,6 +306,7 @@ jobs: HARNESS_E2E_JUDGE_MODEL: ${{ inputs.judge_model }} HARNESS_E2E_JUDGE_PROVIDER: ${{ inputs.judge_provider }} HARNESS_E2E_QUALITY_ADVISORY: ${{ inputs.quality_advisory }} + HARNESS_E2E_CI_SCORE_FLOOR: ${{ inputs.ci_score_floor }} HARNESS_E2E_ENGINE_REVISION: ${{ needs.build.outputs.engine_revision }} run: harness/tests/e2e/run-ci.sh @@ -330,12 +336,13 @@ jobs: SCENARIO: ${{ matrix.scenario }} SUBJECT: ${{ matrix.subject.id }} QUALITY_ADVISORY: ${{ inputs.quality_advisory }} + CI_SCORE_FLOOR: ${{ inputs.ci_score_floor }} run: | { echo "### Harness E2E · $SUBJECT · $SCENARIO" echo if [[ "$QUALITY_ADVISORY" == "true" ]]; then - echo "Quality score gate: advisory. Hard gates and technical failures remain blocking." + echo "CI gate: scores below ${CI_SCORE_FLOOR}% and technical failures are blocking. Higher-scoring quality and hard-gate failures remain visible but do not fail the workflow." else echo "Quality score gate: enforced." fi diff --git a/.github/workflows/harness-e2e-daily.yml b/.github/workflows/harness-e2e-daily.yml index 8646b361f..7436e91b4 100644 --- a/.github/workflows/harness-e2e-daily.yml +++ b/.github/workflows/harness-e2e-daily.yml @@ -69,6 +69,8 @@ jobs: with: runs: 3 max_parallel: 2 + quality_advisory: true + ci_score_floor: 50 source_ref: ${{ needs.context.outputs.source_sha }} benchmark_lane: daily release_tag: daily/${{ needs.context.outputs.benchmark_day }} diff --git a/.github/workflows/harness-e2e-main.yml b/.github/workflows/harness-e2e-main.yml index 02f564d4c..51d65b1a3 100644 --- a/.github/workflows/harness-e2e-main.yml +++ b/.github/workflows/harness-e2e-main.yml @@ -44,6 +44,7 @@ jobs: runs: 1 max_parallel: 2 quality_advisory: true + ci_score_floor: 50 benchmark_lane: main subjects: ${{ vars.HARNESS_E2E_SUBJECTS || '[{"id":"anthropic-sonnet","model":"claude-sonnet-4-6","provider":"anthropic"}]' }} judge_model: ${{ vars.HARNESS_E2E_JUDGE_MODEL || 'claude-sonnet-4-6' }} diff --git a/harness/tests/e2e/README.md b/harness/tests/e2e/README.md index b5c254b83..0bd6e24c0 100644 --- a/harness/tests/e2e/README.md +++ b/harness/tests/e2e/README.md @@ -106,8 +106,11 @@ cargo run -p harness-e2e -- run \ `HARNESS_E2E_OUTPUT` are accepted as environment variables. `--runs` accepts values from 1 through 20. -`--quality-advisory` or `HARNESS_E2E_QUALITY_ADVISORY=true` makes score-only -failures non-blocking. Hard-gate and technical failures remain blocking. +`--quality-advisory` or `HARNESS_E2E_QUALITY_ADVISORY=true` keeps degraded +quality and hard-gate results visible without failing CI when the median score +is at least 50. Scores below that floor and technical failures remain blocking. +Override the floor with `--ci-score-floor` or +`HARNESS_E2E_CI_SCORE_FLOOR`. The runner emits a progress heartbeat every 15 seconds with the active turn, step, pending function count, child-session count, and descendant-tree size. diff --git a/harness/tests/e2e/src/main.rs b/harness/tests/e2e/src/main.rs index 281f6ffde..b64ab8034 100644 --- a/harness/tests/e2e/src/main.rs +++ b/harness/tests/e2e/src/main.rs @@ -80,10 +80,19 @@ struct RunArgs { )] progress_interval_seconds: u64, - /// Keep score-only regressions advisory while still failing hard gates and errors. + /// Keep degraded results above the CI score floor advisory; technical errors still fail. #[arg(long, env = "HARNESS_E2E_QUALITY_ADVISORY", default_value_t = false)] quality_advisory: bool, + /// Minimum median score allowed when the CI quality policy is advisory. + #[arg( + long, + env = "HARNESS_E2E_CI_SCORE_FLOOR", + default_value_t = 50, + value_parser = clap::value_parser!(u8).range(1..=100) + )] + ci_score_floor: u8, + /// Run only the selected scenario. Repeat to select more than one. #[arg(long, value_enum)] scenario: Vec, @@ -142,6 +151,7 @@ fn report(args: ReportArgs) -> Result<()> { async fn run(args: RunArgs) -> Result<()> { let quality_advisory = args.quality_advisory; + let ci_score_floor = args.ci_score_floor; let selected_scenarios = scenarios::selected(&args.scenario); let subject = SubjectConfig { model: args.model, @@ -171,13 +181,16 @@ async fn run(args: RunArgs) -> Result<()> { print!("{}", outcome.report.summary(false)); println!("report: {}", outcome.report_path.display()); - if !outcome.report.passed && !(quality_advisory && !outcome.report.has_non_quality_failure()) { + if !outcome.report.passed + && !(quality_advisory && !outcome.report.fails_ci_gate(ci_score_floor)) + { bail!("E2E suite failed"); } if !outcome.report.passed { tracing::warn!( path = %outcome.report_path.display(), - "E2E score is below threshold; quality gate is advisory" + ci_score_floor, + "E2E result is degraded but remains above the advisory CI score floor" ); return Ok(()); } diff --git a/harness/tests/e2e/src/report.rs b/harness/tests/e2e/src/report.rs index de349c246..67ed2ee0c 100644 --- a/harness/tests/e2e/src/report.rs +++ b/harness/tests/e2e/src/report.rs @@ -409,13 +409,14 @@ impl E2eReport { Ok((report, path)) } - pub fn has_non_quality_failure(&self) -> bool { + pub fn fails_ci_gate(&self, score_floor: u8) -> bool { self.scenarios.is_empty() || self.scenarios.iter().any(|scenario| { - scenario - .runs - .iter() - .any(|run| !matches!(run.status, RunStatus::Passed | RunStatus::QualityFailed)) + scenario.aggregate.technical_failures > 0 + || scenario + .aggregate + .median_score + .is_none_or(|score| score < f64::from(score_floor)) }) } } @@ -629,7 +630,7 @@ mod tests { } #[test] - fn advisory_mode_only_tolerates_quality_failures() { + fn advisory_ci_gate_uses_the_score_floor_for_quality_and_hard_gate_failures() { let quality = E2eReport::new( model(), None, @@ -637,12 +638,29 @@ mod tests { None, vec![aggregate(vec![run(70, false)])], ); - assert!(!quality.has_non_quality_failure()); + assert!(!quality.fails_ci_gate(50)); - let mut hard_gate = run(100, true); + let below_floor = E2eReport::new( + model(), + None, + None, + None, + vec![aggregate(vec![run(49, false)])], + ); + assert!(below_floor.fails_ci_gate(50)); + + let mut hard_gate = run(80, true); hard_gate.status = RunStatus::HardGateFailed; let hard_gate = E2eReport::new(model(), None, None, None, vec![aggregate(vec![hard_gate])]); - assert!(hard_gate.has_non_quality_failure()); + assert!(!hard_gate.fails_ci_gate(50)); + } + + #[test] + fn advisory_ci_gate_keeps_technical_failures_blocking() { + let mut technical = run(80, true); + technical.status = RunStatus::InfrastructureError; + let report = E2eReport::new(model(), None, None, None, vec![aggregate(vec![technical])]); + assert!(report.fails_ci_gate(50)); } fn model() -> ModelArtifact {