(MOT-4277) fix(harness): align CI gating and transcript labels

.github/benchmark-site/execution-transcript.js

@@ -27,6 +27,28 @@
     return String(entry?.entry_id || fallback).replace(/[^a-zA-Z0-9_-]/g, "-");
   }
 
+  function displayFunctionId(functionId, argumentsValue) {
+    const wrappedFunction =
+      functionId === "agent_trigger" &&
+      argumentsValue &&
+      typeof argumentsValue === "object" &&
+      typeof argumentsValue.function === "string"
+        ? argumentsValue.function.trim()
+        : "";
+    return wrappedFunction || functionId || "unknown function";
+  }
+
+  function displayFunctionArguments(functionId, argumentsValue) {
+    if (
+      functionId === "agent_trigger" &&
+      argumentsValue &&
+      typeof argumentsValue === "object"
+    ) {
+      return argumentsValue.payload ?? null;
+    }
+    return argumentsValue ?? null;
+  }
+
   function normalizeTranscript(messages) {
     const events = [];
     const calls = new Map();
@@ -61,8 +83,8 @@
               ),
               kind: "tool",
               callId,
-              functionId: block.function_id || "unknown function",
-              arguments: block.arguments ?? null,
+              functionId: displayFunctionId(block.function_id, block.arguments),
+              arguments: displayFunctionArguments(block.function_id, block.arguments),
               timestamp: message.timestamp || null,
               result: null,
               isError: false,
@@ -160,6 +182,8 @@
 
   return {
     contentBlocks,
+    displayFunctionArguments,
+    displayFunctionId,
     filterTranscript,
     messageText,
     normalizeTranscript,

.github/benchmark-site/execution-transcript.test.cjs

@@ -2,12 +2,34 @@ const test = require("node:test");
 const assert = require("node:assert/strict");
 
 const {
+  displayFunctionArguments,
+  displayFunctionId,
   filterTranscript,
   messageText,
   normalizeTranscript,
   summarizeTranscript,
 } = require("./execution-transcript.js");
 
+test("shows the wrapped function name instead of agent_trigger", () => {
+  const wrappedArguments = {
+    function: "database::query",
+    payload: { db: "primary", sql: "SELECT 1" },
+  };
+  assert.equal(
+    displayFunctionId("agent_trigger", wrappedArguments),
+    "database::query",
+  );
+  assert.deepEqual(displayFunctionArguments("agent_trigger", wrappedArguments), {
+    db: "primary",
+    sql: "SELECT 1",
+  });
+  assert.deepEqual(displayFunctionArguments("state::get", { scope: "test" }), {
+    scope: "test",
+  });
+  assert.equal(displayFunctionId("state::get", { scope: "test" }), "state::get");
+  assert.equal(displayFunctionId("agent_trigger", { payload: {} }), "agent_trigger");
+});
+
 test("extracts readable text from current and legacy message content", () => {
   assert.equal(messageText(" Legacy response. "), " Legacy response. ");
   assert.equal(
@@ -39,8 +61,11 @@ test("pairs function results with calls and marks recovered errors", () => {
           {
             type: "function_call",
             id: "call-ok",
-            function_id: "state::get",
-            arguments: { key: "status" },
+            function_id: "agent_trigger",
+            arguments: {
+              function: "state::get",
+              payload: { key: "status" },
+            },
           },
           {
             type: "function_call",
@@ -85,6 +110,7 @@ test("pairs function results with calls and marks recovered errors", () => {
     ],
   );
   assert.equal(events[2].result.details.value, "ready");
+  assert.deepEqual(events[2].arguments, { key: "status" });
   assert.equal(events[2].status, "completed");
   assert.equal(events[3].isError, true);
   assert.equal(events[3].result.details.error, "command failed");

.github/workflows/_harness-e2e.yml

@@ -14,10 +14,15 @@ on:
         type: number
         default: 2
       quality_advisory:
-        description: Report score failures without failing the workflow
+        description: Only fail degraded results below the CI score floor
         required: false
         type: boolean
         default: false
+      ci_score_floor:
+        description: Minimum median score allowed by the advisory CI policy
+        required: false
+        type: number
+        default: 50
       subjects:
         description: JSON array of subject entries with id, model, and provider
         required: false
@@ -301,6 +306,7 @@ jobs:
           HARNESS_E2E_JUDGE_MODEL: ${{ inputs.judge_model }}
           HARNESS_E2E_JUDGE_PROVIDER: ${{ inputs.judge_provider }}
           HARNESS_E2E_QUALITY_ADVISORY: ${{ inputs.quality_advisory }}
+          HARNESS_E2E_CI_SCORE_FLOOR: ${{ inputs.ci_score_floor }}
           HARNESS_E2E_ENGINE_REVISION: ${{ needs.build.outputs.engine_revision }}
         run: harness/tests/e2e/run-ci.sh
 
@@ -330,12 +336,13 @@ jobs:
           SCENARIO: ${{ matrix.scenario }}
           SUBJECT: ${{ matrix.subject.id }}
           QUALITY_ADVISORY: ${{ inputs.quality_advisory }}
+          CI_SCORE_FLOOR: ${{ inputs.ci_score_floor }}
         run: |
           {
             echo "### Harness E2E · $SUBJECT · $SCENARIO"
             echo
             if [[ "$QUALITY_ADVISORY" == "true" ]]; then
-              echo "Quality score gate: advisory. Hard gates and technical failures remain blocking."
+              echo "CI gate: scores below ${CI_SCORE_FLOOR}% and technical failures are blocking. Higher-scoring quality and hard-gate failures remain visible but do not fail the workflow."
             else
               echo "Quality score gate: enforced."
             fi

.github/workflows/harness-e2e-daily.yml

@@ -69,6 +69,8 @@ jobs:
     with:
       runs: 3
       max_parallel: 2
+      quality_advisory: true
+      ci_score_floor: 50
       source_ref: ${{ needs.context.outputs.source_sha }}
       benchmark_lane: daily
       release_tag: daily/${{ needs.context.outputs.benchmark_day }}

.github/workflows/harness-e2e-main.yml

@@ -44,6 +44,7 @@ jobs:
       runs: 1
       max_parallel: 2
       quality_advisory: true
+      ci_score_floor: 50
       benchmark_lane: main
       subjects: ${{ vars.HARNESS_E2E_SUBJECTS || '[{"id":"anthropic-sonnet","model":"claude-sonnet-4-6","provider":"anthropic"}]' }}
       judge_model: ${{ vars.HARNESS_E2E_JUDGE_MODEL || 'claude-sonnet-4-6' }}

harness/tests/e2e/README.md

@@ -106,8 +106,11 @@ cargo run -p harness-e2e -- run \
 `HARNESS_E2E_OUTPUT` are accepted as environment variables. `--runs` accepts
 values from 1 through 20.
 
-`--quality-advisory` or `HARNESS_E2E_QUALITY_ADVISORY=true` makes score-only
-failures non-blocking. Hard-gate and technical failures remain blocking.
+`--quality-advisory` or `HARNESS_E2E_QUALITY_ADVISORY=true` keeps degraded
+quality and hard-gate results visible without failing CI when the median score
+is at least 50. Scores below that floor and technical failures remain blocking.
+Override the floor with `--ci-score-floor` or
+`HARNESS_E2E_CI_SCORE_FLOOR`.
 
 The runner emits a progress heartbeat every 15 seconds with the active turn,
 step, pending function count, child-session count, and descendant-tree size.

harness/tests/e2e/src/main.rs

@@ -80,10 +80,19 @@ struct RunArgs {
     )]
     progress_interval_seconds: u64,
 
-    /// Keep score-only regressions advisory while still failing hard gates and errors.
+    /// Keep degraded results above the CI score floor advisory; technical errors still fail.
     #[arg(long, env = "HARNESS_E2E_QUALITY_ADVISORY", default_value_t = false)]
     quality_advisory: bool,
 
+    /// Minimum median score allowed when the CI quality policy is advisory.
+    #[arg(
+        long,
+        env = "HARNESS_E2E_CI_SCORE_FLOOR",
+        default_value_t = 50,
+        value_parser = clap::value_parser!(u8).range(1..=100)
+    )]
+    ci_score_floor: u8,
+
     /// Run only the selected scenario. Repeat to select more than one.
     #[arg(long, value_enum)]
     scenario: Vec<ScenarioId>,
@@ -142,6 +151,7 @@ fn report(args: ReportArgs) -> Result<()> {
 
 async fn run(args: RunArgs) -> Result<()> {
     let quality_advisory = args.quality_advisory;
+    let ci_score_floor = args.ci_score_floor;
     let selected_scenarios = scenarios::selected(&args.scenario);
     let subject = SubjectConfig {
         model: args.model,
@@ -171,13 +181,16 @@ async fn run(args: RunArgs) -> Result<()> {
 
     print!("{}", outcome.report.summary(false));
     println!("report: {}", outcome.report_path.display());
-    if !outcome.report.passed && !(quality_advisory && !outcome.report.has_non_quality_failure()) {
+    if !outcome.report.passed
+        && !(quality_advisory && !outcome.report.fails_ci_gate(ci_score_floor))
+    {
         bail!("E2E suite failed");
     }
     if !outcome.report.passed {
         tracing::warn!(
             path = %outcome.report_path.display(),
-            "E2E score is below threshold; quality gate is advisory"
+            ci_score_floor,
+            "E2E result is degraded but remains above the advisory CI score floor"
         );
         return Ok(());
     }

harness/tests/e2e/src/report.rs

@@ -409,13 +409,14 @@ impl E2eReport {
         Ok((report, path))
     }
 
-    pub fn has_non_quality_failure(&self) -> bool {
+    pub fn fails_ci_gate(&self, score_floor: u8) -> bool {
         self.scenarios.is_empty()
             || self.scenarios.iter().any(|scenario| {
-                scenario
-                    .runs
-                    .iter()
-                    .any(|run| !matches!(run.status, RunStatus::Passed | RunStatus::QualityFailed))
+                scenario.aggregate.technical_failures > 0
+                    || scenario
+                        .aggregate
+                        .median_score
+                        .is_none_or(|score| score < f64::from(score_floor))
             })
     }
 }
@@ -629,20 +630,37 @@ mod tests {
     }
 
     #[test]
-    fn advisory_mode_only_tolerates_quality_failures() {
+    fn advisory_ci_gate_uses_the_score_floor_for_quality_and_hard_gate_failures() {
         let quality = E2eReport::new(
             model(),
             None,
             None,
             None,
             vec![aggregate(vec![run(70, false)])],
         );
-        assert!(!quality.has_non_quality_failure());
+        assert!(!quality.fails_ci_gate(50));
 
-        let mut hard_gate = run(100, true);
+        let below_floor = E2eReport::new(
+            model(),
+            None,
+            None,
+            None,
+            vec![aggregate(vec![run(49, false)])],
+        );
+        assert!(below_floor.fails_ci_gate(50));
+
+        let mut hard_gate = run(80, true);
         hard_gate.status = RunStatus::HardGateFailed;
         let hard_gate = E2eReport::new(model(), None, None, None, vec![aggregate(vec![hard_gate])]);
-        assert!(hard_gate.has_non_quality_failure());
+        assert!(!hard_gate.fails_ci_gate(50));
+    }
+
+    #[test]
+    fn advisory_ci_gate_keeps_technical_failures_blocking() {
+        let mut technical = run(80, true);
+        technical.status = RunStatus::InfrastructureError;
+        let report = E2eReport::new(model(), None, None, None, vec![aggregate(vec![technical])]);
+        assert!(report.fails_ci_gate(50));
     }
 
     fn model() -> ModelArtifact {