diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index dec04499970..98ab0a0ab75 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1160,13 +1160,6 @@ jobs: export OPENSHELL_BIN "$OPENSHELL_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts - if [[ "$NEMOCLAW_MCP_BRIDGE_AGENT" == "deepagents" ]]; then - npx vitest run --project e2e-live \ - test/e2e/live/openshell-credential-generation-window.test.ts \ - -t '^openshell-credential-generation-window$' \ - --no-file-parallelism \ - --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - fi - name: Finalize runner comparison telemetry if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }} @@ -1193,6 +1186,117 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + openshell-credential-generation-window: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') || contains(format(',{0},', inputs.jobs), ',openshell-credential-generation-window,') || contains(format(',{0},', inputs.targets), ',openshell-credential-generation-window,') }} + runs-on: ubuntu-latest + permissions: + contents: read + # Keep the credential-generation lifecycle on a fresh runner so it can + # execute in parallel with, and fail independently from, the Deep Agents + # MCP lifecycle without sharing destructive sandbox state. + timeout-minutes: 90 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "openshell-credential-generation-window" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_OPENSHELL_CHANNEL: stable + NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1" + NEMOCLAW_RUN_LIVE_E2E: "1" + OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:f4226253a3525c3832adac5b38b419a0f27d1e915effe565b5885e20f93cd5e9 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + + - name: Install and verify cloudflared prerequisite + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + + - name: Generate MCP test TLS + run: bash test/e2e/setup-mcp-test-tls.sh + + - name: Install OpenShell CLI + env: + NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" + run: | + set -euo pipefail + bash scripts/install-openshell.sh + mkdir -p "$E2E_ARTIFACT_DIR/openshell-credential-generation-window" + jq -n '{ + schemaVersion: 1, + sourceRepository: "NVIDIA/OpenShell", + releaseTag: "v0.0.85", + sourceSha: "3dee5570a46076a57a3b056f35f35ebc0861ac85", + artifacts: { + cli: {binarySha256: "222d9d53a142691d7a7de2c692f38e52d24066f9f633d53746c5fef775861bc8"}, + gateway: {binarySha256: "33bb479d936c3c1b17dd475df05747be9de74564fb67d69a4c33cdd01181d02f"}, + standaloneSandbox: {binarySha256: "863ef21ab7ef623f5e7a8728c4e5532b46bfbae3ace3b800665a1c6353a1f7d2"} + } + }' > "$E2E_ARTIFACT_DIR/openshell-credential-generation-window/openshell-exact-main-provenance.json" + + - name: Run OpenShell credential generation-window live test + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-live \ + test/e2e/live/openshell-credential-generation-window.test.ts \ + -t '^openshell-credential-generation-window$' \ + --no-file-parallelism \ + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts + + - id: credential_window_artifact_secret_scan + name: Scan credential-window artifacts for fixture credentials + if: always() + run: >- + npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts + e2e-artifacts/live/openshell-credential-generation-window + + - name: Upload credential-window artifacts + if: ${{ always() && steps.credential_window_artifact_secret_scan.outcome == 'success' }} + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-openshell-credential-generation-window + path: e2e-artifacts/live/openshell-credential-generation-window/ + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + mcp-bridge-dev: needs: generate-matrix # Moving OpenShell dev artifacts are compatibility evidence only and must @@ -5850,6 +5954,7 @@ jobs: shared-e2e, openshell-gateway-auth-contract, mcp-bridge, + openshell-credential-generation-window, mcp-bridge-dev, skill-agent, openclaw-skill-cli, diff --git a/docs/security/openshell-0.0.85-migration-review.md b/docs/security/openshell-0.0.85-migration-review.md index dfaabcddccd..dab3a8e6888 100644 --- a/docs/security/openshell-0.0.85-migration-review.md +++ b/docs/security/openshell-0.0.85-migration-review.md @@ -551,16 +551,11 @@ usable. The target then reattaches through `mcp restart`, rebuilds without the host MCP secret, and removes the bridge. Every request is identified independently of its credential, and the upstream ledger is required to contain no literal resolve placeholder. -The stable workflow keeps the OpenClaw, Hermes, and Deep Agents MCP lifecycles -on three fresh-runner shards with separate artifacts. It runs the -`openshell-credential-generation-window` live target once, after the Deep Agents -case, rather than repeating that proof in every shard. -The workflow first ran this bounded target against the exact reviewed development -artifacts and scanned its artifacts for the whole generated-secret prefix. The -stable-source review retains the upstream -`expired_retained_generation_does_not_resolve` unit. The default stable job now -binds the target to tag commit `3dee5570`, the extracted release binaries, and the -immutable supervisor index; its final result for the PR SHA remains a merge gate. +The stable workflow keeps the OpenClaw, Hermes, and Deep Agents MCP lifecycles on three fresh-runner shards with separate artifacts. +It runs the `openshell-credential-generation-window` live target once on a fourth fresh runner in parallel with the stable MCP agent matrix, rather than serializing or repeating that proof in the agent matrix. +The workflow first ran this bounded target against the exact reviewed development artifacts and scanned its artifacts for the whole generated-secret prefix. +The stable-source review retains the upstream `expired_retained_generation_does_not_resolve` unit. +The default stable job now binds the target to tag commit `3dee5570`, the extracted release binaries, and the immutable supervisor index; its final result for the PR SHA remains a merge gate. ### v0.0.82 to v0.0.83 diff --git a/scripts/checks/check-cloudflared-update.sh b/scripts/checks/check-cloudflared-update.sh index 8d99d38a416..1a03f0f880d 100755 --- a/scripts/checks/check-cloudflared-update.sh +++ b/scripts/checks/check-cloudflared-update.sh @@ -2,14 +2,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# invalidState: the four reviewed E2E consumers drift to different cloudflared +# invalidState: the five reviewed E2E consumers drift to different cloudflared # versions/digests, or their shared pin no longer matches the upstream asset. -# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all four +# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all five # workflow pins and independently verifies the downloaded bytes. # whyNotSourceFix: upstream cannot enforce which release NemoClaw workflows use. -# regressionTest: cloudflared-update-check-workflow.test.ts covers four-pin +# regressionTest: cloudflared-update-check-workflow.test.ts covers five-pin # parity, asset URL identity, digest mismatch, and update instructions. -# removalCondition: remove this checker when the four consumers share one +# removalCondition: remove this checker when the five consumers share one # machine-readable dependency manifest with equivalent live asset verification. set -euo pipefail @@ -48,10 +48,10 @@ done < <( "${E2E_WORKFLOW}" ) -[[ "${#version_pins[@]}" -eq 4 ]] \ - || fail "expected exactly four CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}" -[[ "${#sha_pins[@]}" -eq 4 ]] \ - || fail "expected exactly four CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}" +[[ "${#version_pins[@]}" -eq 5 ]] \ + || fail "expected exactly five CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}" +[[ "${#sha_pins[@]}" -eq 5 ]] \ + || fail "expected exactly five CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}" pinned_version="${version_pins[0]}" pinned_sha="$(printf '%s' "${sha_pins[0]}" | tr '[:upper:]' '[:lower:]')" @@ -127,7 +127,7 @@ print_update_instructions() { 'Update locations:' \ " ${workflow_display} CLOUDFLARED_VERSION lines: ${version_lines}" \ " ${workflow_display} CLOUDFLARED_DEB_SHA256 lines: ${sha_lines}" \ - 'Set all four version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 + 'Set all five version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 } if [[ "${latest_version}" != "${pinned_version}" ]]; then diff --git a/test/cloudflared-update-check-workflow.test.ts b/test/cloudflared-update-check-workflow.test.ts index 6af84a4e29b..ba996ee8729 100644 --- a/test/cloudflared-update-check-workflow.test.ts +++ b/test/cloudflared-update-check-workflow.test.ts @@ -40,7 +40,7 @@ function pinValues(source: string, name: string): string[] { function writePinFixture(file: string, version: string, sha256: string): void { fs.writeFileSync( file, - ["one", "two", "three", "four"] + ["one", "two", "three", "four", "five"] .map( (job) => ` ${job}:\n env:\n CLOUDFLARED_VERSION: "${version}"\n CLOUDFLARED_DEB_SHA256: "${sha256}"`, @@ -149,11 +149,11 @@ describe("cloudflared update-check workflow contract", () => { expect(checkout?.with?.["persist-credentials"]).toBe(false); }); - it("extracts exactly four identical reviewed version and SHA256 pins", () => { + it("extracts exactly five identical reviewed version and SHA256 pins", () => { const versions = pinValues(e2e, "CLOUDFLARED_VERSION"); const hashes = pinValues(e2e, "CLOUDFLARED_DEB_SHA256"); - expect(versions).toHaveLength(4); - expect(hashes).toHaveLength(4); + expect(versions).toHaveLength(5); + expect(hashes).toHaveLength(5); expect(new Set(versions).size).toBe(1); expect(new Set(hashes).size).toBe(1); expect(versions[0]).toMatch(/^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$/u); @@ -203,7 +203,7 @@ describe("cloudflared update-check workflow contract", () => { ); expect(fixture.result.stderr).toContain("CLOUDFLARED_VERSION lines:"); expect(fixture.result.stderr).toContain("CLOUDFLARED_DEB_SHA256 lines:"); - expect(fixture.result.stderr).toContain("Set all four version/SHA256 pairs"); + expect(fixture.result.stderr).toContain("Set all five version/SHA256 pairs"); } finally { fs.rmSync(fixture.tempDir, { recursive: true, force: true }); } diff --git a/test/e2e/README.md b/test/e2e/README.md index 77373a0d3ee..c81309351c6 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -169,10 +169,14 @@ lanes: - `rebuild-hermes-stale-base`; - the `hermes` and `deepagents` shards of `mcp-bridge`. -The OpenClaw shards of the matrix jobs, the `openclaw` MCP shard, and -`mcp-bridge-dev` remain on `ubuntu-latest`; unrelated jobs retain their -existing runner assignments. Before setting the variable, an organization -owner must: +The OpenClaw shards of the matrix jobs, the `openclaw` MCP shard, +`mcp-bridge-dev`, and `openshell-credential-generation-window` remain on +`ubuntu-latest`; unrelated jobs retain their existing runner assignments. +The credential-generation window runs as an independent fresh-runner job in +parallel with the stable MCP agent matrix. Default full-suite dispatches and +explicit `mcp-bridge` selections run both jobs, while the credential-window job +keeps its own exact-release provenance, secret scan, and artifact. +Before setting the variable, an organization owner must: 1. Create a GitHub-hosted Ubuntu x64 larger runner with 8 vCPU, 32 GB RAM, and 300 GB SSD in a dedicated runner group. diff --git a/test/e2e/support/mcp-workflow-boundary.test.ts b/test/e2e/support/mcp-workflow-boundary.test.ts index 764a57c4134..052f4631234 100644 --- a/test/e2e/support/mcp-workflow-boundary.test.ts +++ b/test/e2e/support/mcp-workflow-boundary.test.ts @@ -74,34 +74,67 @@ describe("MCP workflow artifact boundary", () => { } }); + it("rejects re-serializing the credential generation-window proof behind the MCP matrix", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); + const workflowPath = path.join(directory, "e2e.yaml"); + try { + const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { + jobs: Record< + string, + { + needs?: string | string[]; + steps: Array<{ name?: string; run?: string }>; + } + >; + }; + const mcpRun = workflow.jobs["mcp-bridge"].steps.find( + (step) => step.name === "Run MCP OpenShell provider live test", + ); + requireFixture(mcpRun?.run, "MCP stable lifecycle fixture is missing"); + mcpRun.run += "\ntest/e2e/live/openshell-credential-generation-window.test.ts"; + workflow.jobs["openshell-credential-generation-window"].needs = [ + "generate-matrix", + "mcp-bridge", + ]; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "mcp-bridge must not serialize the independent credential generation-window proof", + "openshell-credential-generation-window must depend only on matrix generation so it can run in parallel", + ]), + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + it.each([ { - expected: "mcp-bridge must isolate the credential generation-window proof to one shard", - mutate: (run: string) => - run.replace('if [[ "$NEMOCLAW_MCP_BRIDGE_AGENT" == "deepagents" ]]; then', "if true; then"), - name: "runs on every shard", - }, - { - expected: "mcp-bridge must run the credential generation-window lifecycle", + expected: "openshell-credential-generation-window must run its exact isolated live proof", mutate: (run: string) => run.replace("test/e2e/live/openshell-credential-generation-window.test.ts", ""), name: "is missing", }, { expected: - "mcp-bridge credential generation-window proof must publish canonical risk-signal evidence", + "openshell-credential-generation-window must publish one canonical risk-signal stream", mutate: (run: string) => run.replace("--reporter=test/e2e/risk-signal-reporter.ts", ""), name: "omits its risk-signal reporter", }, - ])("rejects a credential generation-window proof that $name", ({ expected, mutate, name }) => { + ])("rejects an independent credential generation-window proof that $name", ({ + expected, + mutate, + name, + }) => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); const workflowPath = path.join(directory, "e2e.yaml"); try { const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { jobs: Record }>; }; - const run = workflow.jobs["mcp-bridge"].steps.find( - (step) => step.name === "Run MCP OpenShell provider live test", + const run = workflow.jobs["openshell-credential-generation-window"].steps.find( + (step) => step.name === "Run OpenShell credential generation-window live test", ); requireFixture(run?.run, "MCP stable lifecycle fixture is missing"); const updatedRun = mutate(run.run); @@ -115,6 +148,29 @@ describe("MCP workflow artifact boundary", () => { } }); + it.each([ + "report-to-pr", + "scorecard", + ])("requires %s to wait for the independent credential-window result", (terminalJob) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); + const workflowPath = path.join(directory, "e2e.yaml"); + try { + const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")) as { + jobs: Record; + }; + workflow.jobs[terminalJob].needs = workflow.jobs[terminalJob].needs.filter( + (job) => job !== "openshell-credential-generation-window", + ); + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toContain( + `${terminalJob} must wait for openshell-credential-generation-window`, + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + it("rejects upload action or path drift from the reviewed shared boundary", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); const workflowPath = path.join(directory, "e2e.yaml"); @@ -144,7 +200,10 @@ describe("MCP workflow artifact boundary", () => { } }); - it("rejects an unverified or mutable cloudflared installer in either MCP lane", () => { + it.each([ + "mcp-bridge-dev", + "openshell-credential-generation-window", + ])("rejects an unverified or mutable cloudflared installer in %s", (jobName) => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-workflow-")); const workflowPath = path.join(directory, "e2e.yaml"); try { @@ -160,18 +219,18 @@ describe("MCP workflow artifact boundary", () => { } >; }; - const cloudflared = workflow.jobs["mcp-bridge-dev"].steps.find( + const cloudflared = workflow.jobs[jobName].steps.find( (step) => step.name === "Install and verify cloudflared prerequisite", ); - requireFixture(cloudflared?.env, "MCP cloudflared installer fixture is missing"); + requireFixture(cloudflared?.env, `${jobName} cloudflared installer fixture is missing`); cloudflared.env.CLOUDFLARED_DEB_SHA256 = "mutable"; cloudflared.run = "sudo apt-get install -y cloudflared"; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); expect(validateMcpOpenShellWorkflowBoundary(workflowPath)).toEqual( expect.arrayContaining([ - "mcp-bridge-dev must pin the reviewed cloudflared package checksum", - "mcp-bridge-dev cloudflared installation must not use mutable package repositories", + `${jobName} must pin the reviewed cloudflared package checksum`, + `${jobName} cloudflared installation must not use mutable package repositories`, ]), ); } finally { diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts index aff7e74d07e..056e578173e 100644 --- a/tools/e2e/mcp-workflow-boundary.mts +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -8,6 +8,7 @@ import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-bou const DEFAULT_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; const MCP_JOBS = ["mcp-bridge", "mcp-bridge-dev"] as const; +const CREDENTIAL_WINDOW_JOB = "openshell-credential-generation-window"; const MCP_AGENT_SHARDS = ["openclaw", "hermes", "deepagents"] as const; const MATRIX_AGENT_EXPRESSION = "${{ matrix.agent }}"; const TERMINAL_JOBS = ["report-to-pr", "scorecard"] as const; @@ -18,18 +19,28 @@ const DEV_COMPATIBILITY_STEP_ID = "mcp_runtime_compatibility"; const DEV_COMPATIBILITY_TOOL = "tools/e2e/mcp-bridge-runtime-compatibility.mts"; const CREDENTIAL_WINDOW_ID = "openshell-credential-generation-window"; const CREDENTIAL_WINDOW_FILE = `test/e2e/live/${CREDENTIAL_WINDOW_ID}.test.ts`; -const CREDENTIAL_WINDOW_SHARD = "deepagents"; +const CREDENTIAL_WINDOW_ARTIFACT_DIR = "e2e-artifacts/live/openshell-credential-generation-window"; +const CREDENTIAL_WINDOW_RUN_STEP = "Run OpenShell credential generation-window live test"; +const CREDENTIAL_WINDOW_JOB_CONDITION = + "${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') || contains(format(',{0},', inputs.jobs), ',openshell-credential-generation-window,') || contains(format(',{0},', inputs.targets), ',openshell-credential-generation-window,') }}"; const STABLE_RELEASE_SOURCE_SHA = "3dee5570a46076a57a3b056f35f35ebc0861ac85"; const STABLE_RELEASE_SUPERVISOR_INDEX = "f4226253a3525c3832adac5b38b419a0f27d1e915effe565b5885e20f93cd5e9"; -const STABLE_RELEASE_PROVENANCE_TOKENS = [ +const STABLE_RELEASE_IDENTITY_TOKENS = [ 'releaseTag: "v0.0.85"', STABLE_RELEASE_SOURCE_SHA, "222d9d53a142691d7a7de2c692f38e52d24066f9f633d53746c5fef775861bc8", "33bb479d936c3c1b17dd475df05747be9de74564fb67d69a4c33cdd01181d02f", "863ef21ab7ef623f5e7a8728c4e5532b46bfbae3ace3b800665a1c6353a1f7d2", +] as const; +const STABLE_RELEASE_PROVENANCE_TOKENS = [ + ...STABLE_RELEASE_IDENTITY_TOKENS, "mcp-bridge-deepagents/openshell-exact-main-provenance.json", ] as const; +const CREDENTIAL_WINDOW_PROVENANCE_TOKENS = [ + ...STABLE_RELEASE_IDENTITY_TOKENS, + "openshell-credential-generation-window/openshell-exact-main-provenance.json", +] as const; const DEV_COMPATIBILITY_RUN = [ "set -euo pipefail", 'export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"', @@ -204,7 +215,7 @@ function validateJobIdentity( function validateJobSecurity( errors: string[], - jobName: (typeof MCP_JOBS)[number], + jobName: string, job: UnknownRecord, canonicalDockerAuth: UnknownRecord, ): void { @@ -438,37 +449,11 @@ function validateJobExecution( `${jobName} must publish canonical risk-signal evidence`, ); if (jobName === "mcp-bridge") { - const riskReporter = "--reporter=test/e2e/risk-signal-reporter.ts"; - const reporterCount = asString(run.run).split(riskReporter).length - 1; - if (reporterCount !== 1) { + if (asString(run.run).includes(CREDENTIAL_WINDOW_FILE)) { errors.push( - "mcp-bridge credential generation-window proof must publish canonical risk-signal evidence", + "mcp-bridge must not serialize the independent credential generation-window proof", ); } - requireContains( - errors, - run.run, - `if [[ "$NEMOCLAW_MCP_BRIDGE_AGENT" == "${CREDENTIAL_WINDOW_SHARD}" ]]; then`, - "mcp-bridge must isolate the credential generation-window proof to one shard", - ); - requireContains( - errors, - run.run, - CREDENTIAL_WINDOW_FILE, - "mcp-bridge must run the credential generation-window lifecycle", - ); - requireContains( - errors, - run.run, - `-t '^${CREDENTIAL_WINDOW_ID}$'`, - "mcp-bridge must select the exact credential generation-window proof", - ); - requireContains( - errors, - run.run, - "--no-file-parallelism", - "mcp-bridge must serialize the credential generation-window proof", - ); } requireEqual( errors, @@ -524,6 +509,205 @@ function validateJobExecution( } } +function validateCredentialWindowJob( + errors: string[], + job: UnknownRecord, + canonicalDockerAuth: UnknownRecord, +): void { + if (Object.keys(job).length === 0) { + errors.push(`missing independent MCP job: ${CREDENTIAL_WINDOW_JOB}`); + return; + } + + requireEqual( + errors, + JSON.stringify(jobNeeds(job)), + JSON.stringify(["generate-matrix"]), + `${CREDENTIAL_WINDOW_JOB} must depend only on matrix generation so it can run in parallel`, + ); + requireEqual( + errors, + job["runs-on"], + "ubuntu-latest", + `${CREDENTIAL_WINDOW_JOB} must use its independent standard runner`, + ); + requireEqual( + errors, + job["timeout-minutes"], + 90, + `${CREDENTIAL_WINDOW_JOB} must retain its bounded 90-minute budget`, + ); + requireEqual( + errors, + job.if, + CREDENTIAL_WINDOW_JOB_CONDITION, + `${CREDENTIAL_WINDOW_JOB} must remain default-enabled and follow explicit MCP selections`, + ); + + const env = asRecord(job.env); + const expectedEnv = { + E2E_JOB: "1", + E2E_TARGET_ID: CREDENTIAL_WINDOW_JOB, + E2E_ARTIFACT_DIR: `\${{ github.workspace }}/${CREDENTIAL_WINDOW_ARTIFACT_DIR}`, + NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1", + NEMOCLAW_RUN_LIVE_E2E: "1", + OPENSHELL_DOCKER_SUPERVISOR_IMAGE: `ghcr.io/nvidia/openshell/supervisor@sha256:${STABLE_RELEASE_SUPERVISOR_INDEX}`, + }; + if (!hasExactEntries(env, expectedEnv)) { + errors.push(`${CREDENTIAL_WINDOW_JOB} must use only its reviewed exact-stable environment`); + } + validateJobSecurity(errors, CREDENTIAL_WINDOW_JOB, job, canonicalDockerAuth); + + const steps = asSteps(job); + const prepare = namedStep(job, "Prepare E2E workspace"); + const cloudflared = namedStep(job, "Install and verify cloudflared prerequisite"); + const tls = namedStep(job, "Generate MCP test TLS"); + const install = namedStep(job, "Install OpenShell CLI"); + const run = namedStep(job, CREDENTIAL_WINDOW_RUN_STEP); + const scan = namedStep(job, "Scan credential-window artifacts for fixture credentials"); + const upload = namedStep(job, "Upload credential-window artifacts"); + + if ( + !/^NVIDIA\/NemoClaw\/\.github\/actions\/prepare-e2e@[0-9a-f]{40}$/u.test(asString(prepare.uses)) + ) { + errors.push(`${CREDENTIAL_WINDOW_JOB} must use a SHA-pinned E2E workspace action`); + } + requireEqual( + errors, + asRecord(cloudflared.env).CLOUDFLARED_VERSION, + MCP_CLOUDFLARED_VERSION, + `${CREDENTIAL_WINDOW_JOB} must pin cloudflared ${MCP_CLOUDFLARED_VERSION}`, + ); + requireEqual( + errors, + asRecord(cloudflared.env).CLOUDFLARED_DEB_SHA256, + MCP_CLOUDFLARED_DEB_SHA256, + `${CREDENTIAL_WINDOW_JOB} must pin the reviewed cloudflared package checksum`, + ); + for (const required of [ + "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb", + "sha256sum -c -", + "dpkg-deb -f", + "sudo dpkg -i", + "cloudflared version ${CLOUDFLARED_VERSION}", + ]) { + requireContains( + errors, + cloudflared.run, + required, + `${CREDENTIAL_WINDOW_JOB} cloudflared installation is not immutable and verified`, + ); + } + for (const forbidden of ["pkg.cloudflare.com", "apt-get install", "apt install"]) { + if (asString(cloudflared.run).includes(forbidden)) { + errors.push( + `${CREDENTIAL_WINDOW_JOB} cloudflared installation must not use mutable package repositories`, + ); + } + } + requireEqual( + errors, + tls.run, + "bash test/e2e/setup-mcp-test-tls.sh", + `${CREDENTIAL_WINDOW_JOB} must generate its HTTPS fixture before installation`, + ); + requireEqual( + errors, + asRecord(install.env).NEMOCLAW_OPENSHELL_FORCE_INSTALL, + "1", + `${CREDENTIAL_WINDOW_JOB} must force the stable OpenShell install`, + ); + requireContains( + errors, + install.run, + "bash scripts/install-openshell.sh", + `${CREDENTIAL_WINDOW_JOB} must use the repository OpenShell installer`, + ); + for (const token of CREDENTIAL_WINDOW_PROVENANCE_TOKENS) { + requireContains( + errors, + install.run, + token, + `${CREDENTIAL_WINDOW_JOB} stable release provenance is missing reviewed identity: ${token}`, + ); + } + + for (const required of [ + CREDENTIAL_WINDOW_FILE, + `-t '^${CREDENTIAL_WINDOW_ID}$'`, + "--no-file-parallelism", + "--reporter=test/e2e/risk-signal-reporter.ts", + ]) { + requireContains( + errors, + run.run, + required, + `${CREDENTIAL_WINDOW_JOB} must run its exact isolated live proof`, + ); + } + const riskReporter = "--reporter=test/e2e/risk-signal-reporter.ts"; + if (asString(run.run).split(riskReporter).length - 1 !== 1) { + errors.push(`${CREDENTIAL_WINDOW_JOB} must publish one canonical risk-signal stream`); + } + + requireEqual( + errors, + scan.id, + "credential_window_artifact_secret_scan", + `${CREDENTIAL_WINDOW_JOB} secret scanner must expose its gated step id`, + ); + requireEqual( + errors, + scan.if, + "always()", + `${CREDENTIAL_WINDOW_JOB} artifact secret scan must run unconditionally`, + ); + for (const required of [ + "tools/e2e/assert-mcp-artifact-secrets-absent.mts", + CREDENTIAL_WINDOW_ARTIFACT_DIR, + ]) { + requireContains( + errors, + scan.run, + required, + `${CREDENTIAL_WINDOW_JOB} artifact secret scan is incomplete`, + ); + } + requireEqual( + errors, + upload.uses, + UPLOAD_E2E_ARTIFACTS_ACTION, + `${CREDENTIAL_WINDOW_JOB} artifact upload must use the reviewed shared uploader`, + ); + requireEqual( + errors, + upload.if, + "${{ always() && steps.credential_window_artifact_secret_scan.outcome == 'success' }}", + `${CREDENTIAL_WINDOW_JOB} artifact upload must be gated by the secret scanner`, + ); + const uploadOptions = asRecord(upload.with); + if ( + !hasExactEntries(uploadOptions, { + name: `e2e-${CREDENTIAL_WINDOW_JOB}`, + path: `${CREDENTIAL_WINDOW_ARTIFACT_DIR}/`, + }) + ) { + errors.push(`${CREDENTIAL_WINDOW_JOB} upload must use exactly its scanned artifact directory`); + } + + const orderedSteps = [prepare, cloudflared, tls, install, run, scan, upload]; + if ( + orderedSteps.some((step) => steps.indexOf(step) < 0) || + orderedSteps.some( + (step, index) => index > 0 && steps.indexOf(step) <= steps.indexOf(orderedSteps[index - 1]!), + ) + ) { + errors.push(`${CREDENTIAL_WINDOW_JOB} must preserve its reviewed execution and scan order`); + } +} + export function validateMcpOpenShellWorkflowBoundary( workflowPath = DEFAULT_WORKFLOW_PATH, ): string[] { @@ -568,11 +752,12 @@ export function validateMcpOpenShellWorkflowBoundary( validateJobSecurity(errors, jobName, job, canonicalDockerAuth); validateJobExecution(errors, jobName, job); } + validateCredentialWindowJob(errors, asRecord(jobs[CREDENTIAL_WINDOW_JOB]), canonicalDockerAuth); for (const terminalJobName of TERMINAL_JOBS) { const terminal = asRecord(jobs[terminalJobName]); const terminalNeeds = new Set(jobNeeds(terminal)); - for (const mcpJob of MCP_JOBS) { + for (const mcpJob of [...MCP_JOBS, CREDENTIAL_WINDOW_JOB]) { if (!terminalNeeds.has(mcpJob)) { errors.push(`${terminalJobName} must wait for ${mcpJob}`); } diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 66933d2e1cd..c0142ad13da 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -35,6 +35,8 @@ const CALLER_ALWAYS = "always()"; const RETIRED_SELECTOR_COMPATIBILITY_JOB = "retired-selector-compatibility"; const MCP_SCANNED_UPLOAD_CONDITION = "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}"; +const CREDENTIAL_WINDOW_SCANNED_UPLOAD_CONDITION = + "${{ always() && steps.credential_window_artifact_secret_scan.outcome == 'success' }}"; const GATEWAY_AUTH_SCANNED_UPLOAD_CONDITION = "${{ always() && steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path != '' }}"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; @@ -231,6 +233,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/", }, ], + [ + "openshell-credential-generation-window", + { + name: "e2e-openshell-credential-generation-window", + path: "e2e-artifacts/live/openshell-credential-generation-window/", + }, + ], ]); const EXPLICIT_CALLER_CONDITIONS = new Map([ @@ -238,6 +247,7 @@ const EXPLICIT_CALLER_CONDITIONS = new Map([ ["staging-brev-launchable", "${{ always() && steps.workspace.outputs.work_dir != '' }}"], ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], + ["openshell-credential-generation-window", CREDENTIAL_WINDOW_SCANNED_UPLOAD_CONDITION], ["openshell-gateway-auth-contract", GATEWAY_AUTH_SCANNED_UPLOAD_CONDITION], ]); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index e2035913b52..061dbadd1b5 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -146,6 +146,7 @@ const COMMON_SECRET_ENV_NAMES = [ const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set([ "hermes-e2e", "hermes-gpu-startup", + "openshell-credential-generation-window", "staging-brev-launchable", ]); const ADAPTER_MANAGED_INFERENCE_JOBS = new Set(["hermes-e2e"]);