From c646c04edf70bb9b127e76f39f3a183a8afc5329 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 07:24:37 -0700 Subject: [PATCH 1/3] fix(ci): support mode-sharded E2E evidence Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 2 ++ test/pr-e2e-gate.test.ts | 4 ++++ .../inference-switch-workflow-boundary.mts | 1 + tools/e2e/pr-e2e-gate.mts | 23 ++++++++++++++++--- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index bd556b4e7e0..5d731663c44 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1175,6 +1175,7 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: "hermes" + NEMOCLAW_E2E_SHARD: ${{ matrix.mode }} NEMOCLAW_SANDBOX_NAME: ${{ matrix.sandbox_name }} NEMOCLAW_SWITCH_PROVIDER: ${{ matrix.switch_provider }} NEMOCLAW_SWITCH_MODEL: ${{ matrix.switch_model }} @@ -4262,6 +4263,7 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: "openclaw" + NEMOCLAW_E2E_SHARD: ${{ matrix.mode }} NEMOCLAW_SANDBOX_NAME: ${{ matrix.sandbox_name }} NEMOCLAW_SWITCH_PROVIDER: ${{ matrix.switch_provider }} NEMOCLAW_SWITCH_MODEL: ${{ matrix.switch_model }} diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index 70fd9071b28..5b01fcf3e6f 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -372,6 +372,10 @@ describe("PR E2E controller", () => { expect(expectedSignalShards(["docs-validation"])).toEqual({ "docs-validation": ["default"], }); + expect(expectedSignalShards(["hermes-inference-switch", "openclaw-inference-switch"])).toEqual({ + "hermes-inference-switch": ["hosted", "anthropic"], + "openclaw-inference-switch": ["hosted", "anthropic"], + }); const broadPlan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: BROAD_FILES }); const broadShards = expectedSignalShards(riskPlanRequiredJobIds(broadPlan)); expect(Object.keys(broadShards)).toHaveLength(13); diff --git a/tools/e2e/inference-switch-workflow-boundary.mts b/tools/e2e/inference-switch-workflow-boundary.mts index ab683543ada..662b75b1fce 100644 --- a/tools/e2e/inference-switch-workflow-boundary.mts +++ b/tools/e2e/inference-switch-workflow-boundary.mts @@ -93,6 +93,7 @@ function validateJob(errors: string[], spec: JobSpec, job: WorkflowJob): void { const requiredEnv: Record = { E2E_ARTIFACT_DIR: `\${{ github.workspace }}/e2e-artifacts/live/${spec.scenario}/\${{ matrix.mode }}`, NEMOCLAW_AGENT: spec.agent, + NEMOCLAW_E2E_SHARD: "${{ matrix.mode }}", NEMOCLAW_SANDBOX_NAME: "${{ matrix.sandbox_name }}", NEMOCLAW_SWITCH_PROVIDER: "${{ matrix.switch_provider }}", NEMOCLAW_SWITCH_MODEL: "${{ matrix.switch_model }}", diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index 687c5b64dc0..bb24d624ba4 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -1433,11 +1433,28 @@ export function expectedSignalShards( throw new Error(`${jobId} matrix agent values must be strings`); } } else if (keys.length === 1 && Array.isArray(matrix.include)) { + const env = isObjectRecord(job.env) ? job.env : {}; + const configuredShard = env.NEMOCLAW_E2E_SHARD; + let shardKey = "agent"; + if (configuredShard !== undefined) { + const match = + typeof configuredShard === "string" + ? /^\$\{\{\s*matrix\.([A-Za-z][A-Za-z0-9_]*)\s*\}\}$/u.exec(configuredShard) + : null; + if (!match) { + throw new Error(`${jobId} NEMOCLAW_E2E_SHARD must name one matrix include field`); + } + shardKey = match[1]!; + } shards = matrix.include.map((entry) => { - if (!isObjectRecord(entry) || typeof entry.agent !== "string") { - throw new Error(`${jobId} matrix include entries must name an agent`); + if (!isObjectRecord(entry) || !Object.hasOwn(entry, shardKey)) { + throw new Error(`${jobId} matrix include entries must name a ${shardKey} shard`); + } + const shard = entry[shardKey]; + if (typeof shard !== "string") { + throw new Error(`${jobId} matrix include entries must name a ${shardKey} shard`); } - return entry.agent; + return shard; }); } else { throw new Error(`${jobId} uses an unsupported evidence matrix`); From 2c8a5179c29f1742ffcebae36faaa79864a65a5e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 08:00:09 -0700 Subject: [PATCH 2/3] test(ci): reject malformed E2E shard selectors Signed-off-by: Aaron Erickson --- ci/source-shape-test-budget.json | 5 +++ test/pr-e2e-gate-shards.test.ts | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 test/pr-e2e-gate-shards.test.ts diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 49b3c98cbe2..28831594717 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -326,6 +326,11 @@ "test": "pins both controller checkouts and installs without lifecycle scripts or caches", "category": "security" }, + { + "file": "test/pr-e2e-gate-shards.test.ts", + "test": "rejects malformed configured matrix shard selectors", + "category": "security" + }, { "file": "test/pr-review-advisor-workflow-boundary.test.ts", "test": "rejects deleting or weakening analysis-workspace symlink removal", diff --git a/test/pr-e2e-gate-shards.test.ts b/test/pr-e2e-gate-shards.test.ts new file mode 100644 index 00000000000..20944158d08 --- /dev/null +++ b/test/pr-e2e-gate-shards.test.ts @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; +import { expectedSignalShards } from "../tools/e2e/pr-e2e-gate.mts"; + +const temporaryWorkflowDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryWorkflowDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +function temporaryE2eWorkflow(source: string): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-shards-")); + temporaryWorkflowDirectories.push(directory); + const workflowPath = path.join(directory, "e2e.yaml"); + fs.writeFileSync(workflowPath, source, "utf8"); + return workflowPath; +} + +describe("PR E2E shard policy", () => { + // source-shape-contract: security -- Malformed matrix shard selectors must fail closed before exact-SHA evidence dispatch + it("rejects malformed configured matrix shard selectors", () => { + const workflow = fs.readFileSync(".github/workflows/e2e.yaml", "utf8"); + const shardExpression = "NEMOCLAW_E2E_SHARD: ${{ matrix.mode }}"; + + const invalidExpression = temporaryE2eWorkflow( + workflow.replace(shardExpression, "NEMOCLAW_E2E_SHARD: mode"), + ); + expect(() => expectedSignalShards(["hermes-inference-switch"], invalidExpression)).toThrow( + /must name one matrix include field/u, + ); + + const missingField = temporaryE2eWorkflow( + workflow.replace(shardExpression, "NEMOCLAW_E2E_SHARD: ${{ matrix.missing }}"), + ); + expect(() => expectedSignalShards(["hermes-inference-switch"], missingField)).toThrow( + /must name a missing shard/u, + ); + + const nonStringField = temporaryE2eWorkflow( + workflow.replace( + " - mode: hosted\n sandbox_name: e2e-hermes-inference-switch", + " - mode: 1\n sandbox_name: e2e-hermes-inference-switch", + ), + ); + expect(() => expectedSignalShards(["hermes-inference-switch"], nonStringField)).toThrow( + /must name a mode shard/u, + ); + }); +}); From f8165ec8528e3f7ef106228023a3b49132fcbc96 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 08:15:37 -0700 Subject: [PATCH 3/3] test(ci): cover inference shard workflow boundary Signed-off-by: Aaron Erickson --- .../inference-switch-workflow-boundary.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/e2e/support/inference-switch-workflow-boundary.test.ts b/test/e2e/support/inference-switch-workflow-boundary.test.ts index 428deec3568..327f154d7fb 100644 --- a/test/e2e/support/inference-switch-workflow-boundary.test.ts +++ b/test/e2e/support/inference-switch-workflow-boundary.test.ts @@ -57,6 +57,20 @@ describe("inference switch workflow boundary", () => { ); }); + it("rejects missing or misconfigured E2E shard mappings", () => { + const missingShard = readInferenceSwitchWorkflow(); + delete missingShard.jobs["hermes-inference-switch"].env!.NEMOCLAW_E2E_SHARD; + expect(validateInferenceSwitchWorkflow(missingShard)).toContain( + "hermes-inference-switch must map NEMOCLAW_E2E_SHARD from its mode matrix", + ); + + const hardcodedShard = readInferenceSwitchWorkflow(); + hardcodedShard.jobs["openclaw-inference-switch"].env!.NEMOCLAW_E2E_SHARD = "hosted"; + expect(validateInferenceSwitchWorkflow(hardcodedShard)).toContain( + "openclaw-inference-switch must map NEMOCLAW_E2E_SHARD from its mode matrix", + ); + }); + it("uses a healthy hosted switch target and scopes its credentials to hosted mode", () => { const wrongTarget = readInferenceSwitchWorkflow(); const hosted = wrongTarget.jobs["hermes-inference-switch"].strategy?.matrix?.include?.find(