ci(e2e): extend Hermes runner telemetry

.github/workflows/e2e.yaml

@@ -1457,6 +1457,11 @@ jobs:
       - *dockerhub-auth
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
+      - name: Initialize runner comparison telemetry
+        if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts initialize
       - name: Install OpenShell CLI
         run: bash scripts/install-openshell.sh
       - name: Run agent turn latency live Vitest test
@@ -1479,6 +1484,11 @@ jobs:
           echo "Using OPENSHELL_BIN=$OPENSHELL_BIN"
           "$OPENSHELL_BIN" --version
           npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/agent-turn-latency.test.ts
+      - name: Finalize runner comparison telemetry
+        if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts finalize
       - name: Upload agent turn latency artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
@@ -5157,6 +5167,12 @@ jobs:
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
 
+      - name: Initialize runner comparison telemetry
+        if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts initialize
+
       - name: Run Bedrock Runtime compatible Anthropic live test
         # Direct Vitest coverage for
         # Preserves the
@@ -5167,6 +5183,12 @@ jobs:
           set -euo pipefail
           npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts
 
+      - name: Finalize runner comparison telemetry
+        if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts finalize
+
       - name: Upload Bedrock Runtime compatible Anthropic artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

test/e2e/README.md

@@ -181,9 +181,11 @@ graph as the live targets:
 ### Runner comparison telemetry
 
 Trusted `main` runs without an alternate checkout SHA record runner-comparison
-telemetry for the #7145 contract: 12 routed workflow lane identities / 15
+telemetry for 14 routed workflow lane identities / 17
 concrete job executions.
 
+- `agent-turn-latency`, spanning its sequential OpenClaw and Hermes setup
+- `bedrock-runtime-compatible-anthropic` with the `hermes` shard
 - `common-egress-agent` with the `openclaw-balanced-weather`,
   `openclaw-open-reference`, and `hermes-open-reference` shards
 - `rebuild-hermes`
@@ -201,7 +203,12 @@ concrete job executions.
 The three extra executions come from `common-egress-agent`, which runs three
 scenario shards, and `hermes-inference-switch`, which runs both listed modes.
 The OpenClaw matrix entries for `mcp-bridge`,
-`channels-stop-start`, and `security-posture` are not instrumented.
+`channels-stop-start`, `security-posture`, and
+`bedrock-runtime-compatible-anthropic` are not instrumented.
+The #7145 standard-versus-larger-runner cohort compares the same lane and
+equivalent workload while varying the runner class. The newly instrumented
+`agent-turn-latency` and Bedrock Hermes lanes extend diagnostic coverage; this
+change does not route them to a larger runner.
 
 Each execution writes one bounded, ordered v2 time series to the canonical
 `runner-comparison.jsonl` ledger. It contains:

test/e2e/support/runner-comparison-workflow-boundary.test.ts

@@ -29,6 +29,8 @@ type Workflow = {
 };
 
 const JOBS = [
+  "agent-turn-latency",
+  "bedrock-runtime-compatible-anthropic",
   "channels-stop-start",
   "common-egress-agent",
   "hermes-dashboard",
@@ -59,8 +61,8 @@ function telemetrySteps(workflow: Workflow, jobId: string): WorkflowStep[] {
   );
 }
 
-describe("runner comparison E2E workflow boundary (#7145)", () => {
-  it("accepts 12 routed workflow lane identities / 15 concrete job executions", () => {
+describe("runner comparison E2E workflow boundary (#7140)", () => {
+  it("accepts 14 routed workflow lane identities / 17 concrete job executions", () => {
     const workflow = loadWorkflow();
 
     expect(validateRunnerComparisonWorkflowBoundary(workflow)).toEqual([]);
@@ -86,12 +88,16 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
     const routedLanes = JOBS.length - 1 + mcpLanes;
     const concreteExecutions =
       routedLanes + commonEgressScenarios!.length - 1 + inferenceSwitchModes!.length - 1;
-    expect(routedLanes).toBe(12);
-    expect(concreteExecutions).toBe(15);
+    expect(routedLanes).toBe(14);
+    expect(concreteExecutions).toBe(17);
   });
 
-  it("locks the matrix topology that produces fifteen concrete executions", () => {
+  it("locks the matrix topology that produces seventeen concrete executions", () => {
     const workflow = loadWorkflow();
+    workflow.jobs["bedrock-runtime-compatible-anthropic"]!.strategy!.matrix!.agent = [
+      "openclaw",
+      "openclaw",
+    ];
     workflow.jobs["mcp-bridge"]!.strategy!.matrix!.agent = ["openclaw", "hermes", "hermes"];
     workflow.jobs["channels-stop-start"]!.strategy!.matrix!.agent = ["openclaw", "openclaw"];
     workflow.jobs["common-egress-agent"]!.strategy!.matrix!.include = [
@@ -110,6 +116,7 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
 
     expect(validateRunnerComparisonWorkflow(workflow)).toEqual(
       expect.arrayContaining([
+        "bedrock-runtime-compatible-anthropic matrix must contain exactly openclaw, hermes for runner comparison telemetry",
         "channels-stop-start matrix must contain exactly openclaw, hermes for runner comparison telemetry",
         "common-egress-agent matrix must contain exactly openclaw-balanced-weather, openclaw-open-reference, hermes-open-reference for runner comparison telemetry",
         "mcp-bridge matrix must contain exactly openclaw, hermes, deepagents for runner comparison telemetry",
@@ -119,7 +126,7 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
     );
   });
 
-  it("rejects runner comparison consumers outside the eleven comparison jobs", () => {
+  it("rejects runner comparison consumers outside the thirteen comparison jobs", () => {
     const workflow = loadWorkflow();
     workflow.jobs["shields-config"]!.steps.push(
       structuredClone(telemetrySteps(workflow, "common-egress-agent")[0]!),
@@ -236,6 +243,11 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
           "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')",
         );
       }
+      const bedrockComparison = step(workflow, "bedrock-runtime-compatible-anthropic", name);
+      bedrockComparison.if = bedrockComparison.if!.replace(
+        "matrix.agent == 'hermes'",
+        "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')",
+      );
     }
 
     expect(validateRunnerComparisonWorkflow(workflow)).toEqual(
@@ -246,6 +258,8 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
         "channels-stop-start must use the exact always-run trusted finalize telemetry step",
         "security-posture must use the exact trusted initialize telemetry step",
         "security-posture must use the exact always-run trusted finalize telemetry step",
+        "bedrock-runtime-compatible-anthropic must use the exact trusted initialize telemetry step",
+        "bedrock-runtime-compatible-anthropic must use the exact always-run trusted finalize telemetry step",
       ]),
     );
   });

tools/e2e/runner-comparison-workflow-boundary.mts

@@ -22,6 +22,14 @@ const MCP_INITIALIZE_GUARD = `\${{ ${TRUSTED_MAIN_GUARD} && ${MCP_AGENT_GUARD} }
 const MCP_FINALIZE_GUARD = `\${{ always() && ${TRUSTED_MAIN_GUARD} && ${MCP_AGENT_GUARD} }}`;
 
 const COMPARISON_JOBS: ReadonlyMap<string, { initializeIf: string; finalizeIf: string }> = new Map([
+  [
+    "agent-turn-latency",
+    { initializeIf: ORDINARY_INITIALIZE_GUARD, finalizeIf: ORDINARY_FINALIZE_GUARD },
+  ],
+  [
+    "bedrock-runtime-compatible-anthropic",
+    { initializeIf: HERMES_INITIALIZE_GUARD, finalizeIf: HERMES_FINALIZE_GUARD },
+  ],
   [
     "channels-stop-start",
     { initializeIf: HERMES_INITIALIZE_GUARD, finalizeIf: HERMES_FINALIZE_GUARD },
@@ -136,16 +144,20 @@ function publicationIndex(jobSteps: readonly WorkflowStep[]): number {
 }
 
 /**
- * Keep the #7145 comparison to 12 routed workflow lane identities / 15
- * concrete trusted-main job executions. Telemetry is best-effort, but it must
- * span the complete stable-capacity job and finish before evidence is scanned
- * or uploaded. Rebuild jobs establish their fixed swap capacity first because
- * the v2 ledger rejects capacity changes after initialization.
+ * Keep runner diagnostics to 14 routed workflow lane identities / 17 concrete
+ * trusted-main job executions. Telemetry is best-effort, but it must span the
+ * complete stable-capacity job and finish before evidence is scanned or
+ * uploaded. Rebuild jobs establish their fixed swap capacity first because the
+ * v2 ledger rejects capacity changes after initialization.
  */
 export function validateRunnerComparisonWorkflow(workflowValue: unknown): string[] {
   const jobs = record(record(workflowValue).jobs);
   const errors: string[] = [];
 
+  requireExactMatrixValues(errors, jobs, "bedrock-runtime-compatible-anthropic", "agent", [
+    "openclaw",
+    "hermes",
+  ]);
   requireExactMatrixValues(errors, jobs, "channels-stop-start", "agent", ["openclaw", "hermes"]);
   requireExactMatrixValues(errors, jobs, "common-egress-agent", "scenario", [
     "openclaw-balanced-weather",