diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2bb9473fe8a..dc1df361688 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -191,6 +191,7 @@ jobs: timeout-minutes: 10 outputs: cli_artifact_provenance: ${{ steps.record_cli_artifact.outputs.provenance }} + e2e_credentials_allowed: ${{ steps.e2e_credentials.outputs.allowed }} dcode_base_contract: ${{ needs.base-image-publication.outputs.dcode_base_contract }} dcode_base_ref: ${{ needs.base-image-publication.outputs.dcode_base_ref }} matrix: ${{ steps.matrix.outputs.matrix }} @@ -563,6 +564,33 @@ jobs: [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before execution" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; } + - id: e2e_credentials + name: Authorize E2E credentials + if: ${{ inputs.checkout_sha != '' }} + env: + CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + REF: ${{ github.ref }} + WORKFLOW_REPOSITORY: ${{ github.repository }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash + run: | + set -euo pipefail + credentials_allowed=false + if [[ "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw" && + "$CHECKOUT_REPOSITORY" == "$WORKFLOW_REPOSITORY" && + "$EVENT_NAME" == "workflow_dispatch" && + "$REF" == "refs/heads/main" && + "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && + "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && + "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" && + "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]; then + credentials_allowed=true + fi + printf 'allowed=%s\n' "$credentials_allowed" >> "$GITHUB_OUTPUT" + - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 @@ -1180,7 +1208,7 @@ jobs: github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} - trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} + trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -1219,7 +1247,7 @@ jobs: github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} - trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} + trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -1259,7 +1287,7 @@ jobs: github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} - trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} + trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -1299,7 +1327,7 @@ jobs: github_token: true shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} - trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} + trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -1339,7 +1367,7 @@ jobs: github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} - trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} + trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 5b8a63d73a8..9f751d03486 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -252,6 +252,123 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); }); + it("rejects changes that bypass E2E credential authorization (#9047)", () => { + const workflow = readE2eOperationsWorkflow(); + delete workflow.jobs["generate-matrix"].outputs!.e2e_credentials_allowed; + const credentialAuthorization = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authorize E2E credentials", + )!; + credentialAuthorization.run = "printf 'allowed=true\\n' >> \"$GITHUB_OUTPUT\""; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "Manual PR credential authorization must expose only the authorization result", + 'Manual PR credential authorization must retain "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw"', + 'Manual PR credential authorization must retain "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', + ]), + ); + }); + + it.each([ + { + caseName: "matching repository and requested SHAs", + checkoutRepository: "NVIDIA/NemoClaw", + workflowRepository: "NVIDIA/NemoClaw", + checkoutShaMatches: true, + workflowShaMatches: true, + expectedAllowed: true, + }, + { + caseName: "a checkout repository outside NVIDIA/NemoClaw", + checkoutRepository: "contributor/NemoClaw", + workflowRepository: "NVIDIA/NemoClaw", + checkoutShaMatches: true, + workflowShaMatches: true, + expectedAllowed: false, + }, + { + caseName: "a workflow repository outside NVIDIA/NemoClaw", + checkoutRepository: "NVIDIA/NemoClaw", + workflowRepository: "contributor/NemoClaw", + checkoutShaMatches: true, + workflowShaMatches: true, + expectedAllowed: false, + }, + { + caseName: "checkout_sha differs from the checked-out commit", + checkoutRepository: "NVIDIA/NemoClaw", + workflowRepository: "NVIDIA/NemoClaw", + checkoutShaMatches: false, + workflowShaMatches: true, + expectedAllowed: false, + }, + { + caseName: "a requested workflow SHA that differs from the running workflow", + checkoutRepository: "NVIDIA/NemoClaw", + workflowRepository: "NVIDIA/NemoClaw", + checkoutShaMatches: true, + workflowShaMatches: false, + expectedAllowed: false, + }, + ])( + "sets E2E credential access to $expectedAllowed for $caseName (#9047)", + ({ + checkoutRepository, + workflowRepository, + checkoutShaMatches, + workflowShaMatches, + expectedAllowed, + }) => { + const workflow = readE2eOperationsWorkflow(); + const credentialAuthorization = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authorize E2E credentials", + )!; + const checkedOutSha = spawnSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", + }).stdout.trim(); + const checkoutSha = checkoutShaMatches ? checkedOutSha : "0".repeat(40); + const workflowSha = "c".repeat(40); + const expectedWorkflowSha = workflowShaMatches ? workflowSha : "d".repeat(40); + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-e2e-credentials-")); + const output = join(directory, "output"); + + try { + writeFileSync(output, ""); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-e", + "-o", + "pipefail", + "-c", + credentialAuthorization.run!, + ], + { + encoding: "utf8", + env: { + ...process.env, + CHECKOUT_REPOSITORY: checkoutRepository, + CHECKOUT_SHA: checkoutSha, + EVENT_NAME: "workflow_dispatch", + EXPECTED_WORKFLOW_SHA: expectedWorkflowSha, + GITHUB_OUTPUT: output, + REF: "refs/heads/main", + WORKFLOW_REPOSITORY: workflowRepository, + WORKFLOW_SHA: workflowSha, + }, + }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(output, "utf8")).toBe(`allowed=${expectedAllowed ? "true" : "false"}\n`); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }, + ); + it("keeps catalogue-owned GPU targets out of the handwritten workflow jobs", () => { const workflow = readE2eOperationsWorkflow(); workflow.jobs["llama-cpp-generic-gpu"] = { diff --git a/test/e2e/support/standard-profile-workflow-boundary.test.ts b/test/e2e/support/standard-profile-workflow-boundary.test.ts index 2d3e135dcc6..211f28286bb 100644 --- a/test/e2e/support/standard-profile-workflow-boundary.test.ts +++ b/test/e2e/support/standard-profile-workflow-boundary.test.ts @@ -54,6 +54,18 @@ describe("standard E2E execution profile boundary", () => { ); }); + it("rejects catalogue callers that bypass E2E credential authorization (#9047)", () => { + const workflow = readWorkflow() as { + jobs: Record }>; + }; + workflow.jobs["catalogue-brave-nvidia-inference"]!.with.trusted_main = + "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }}"; + + expect(validateStandardProfileWorkflowBoundary(workflow)).toContain( + "catalogue-brave-nvidia-inference must pass trusted_main from the catalogue matrix", + ); + }); + it("rejects catalogue callers without dispatch-bound manual PR risk-signal identity", () => { const workflow = readWorkflow() as { jobs: Record }>; @@ -336,7 +348,7 @@ describe("standard E2E execution profile boundary", () => { expect(validateStandardProfileWorkflowBoundary(readWorkflow(), profilePath)).toEqual( expect.arrayContaining([ "standard E2E profile checkout action must use a full commit SHA", - "standard E2E profile must check out the exact candidate without credentials", + "standard E2E profile must check out checkout_sha without credentials", "standard E2E profile Docker Hub auth-required must be guarded by trusted_main", "standard E2E profile must install only the planned host packages with the reviewed action", "standard E2E profile must install host dependencies before workspace prep", diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index ddfd299c4bf..2b6819f68f1 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -56,6 +56,7 @@ const NEEDS_INTERPOLATION = /\$\{\{\s*toJSON\s*\(\s*needs\s*\)\s*\}\}/iu; type WorkflowStep = { "continue-on-error"?: boolean; env?: Record; + id?: string; if?: string; name?: string; run?: string; @@ -260,15 +261,20 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow ); const checkoutIndex = steps.findIndex((step) => step.uses?.startsWith("actions/checkout@")); const validationIndex = steps.findIndex((step) => step.name === "Validate manual PR checkout"); + const credentialAuthorizationIndex = steps.findIndex( + (step) => step.name === "Authorize E2E credentials", + ); const prepareIndex = steps.findIndex((step) => step.name === "Prepare E2E workspace"); if ( authenticationIndex < 0 || checkoutIndex < 0 || validationIndex < 0 || + credentialAuthorizationIndex < 0 || prepareIndex < 0 || authenticationIndex >= checkoutIndex || checkoutIndex >= validationIndex || - validationIndex >= prepareIndex + validationIndex >= credentialAuthorizationIndex || + credentialAuthorizationIndex >= prepareIndex ) { errors.push("Manual PR authorization and validation must surround checkout before preparation"); } @@ -353,6 +359,55 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } } + const credentialAuthorization = + credentialAuthorizationIndex >= 0 ? steps[credentialAuthorizationIndex] : {}; + if ( + matrixJob.outputs?.e2e_credentials_allowed !== + "${{ steps.e2e_credentials.outputs.allowed }}" || + credentialAuthorization.id !== "e2e_credentials" || + credentialAuthorization.if !== "${{ inputs.checkout_sha != '' }}" || + credentialAuthorization.shell !== "bash" + ) { + errors.push("Manual PR credential authorization must expose only the authorization result"); + } + const expectedCredentialAuthorizationEnvironment = { + CHECKOUT_REPOSITORY: "${{ inputs.checkout_repository }}", + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + EVENT_NAME: "${{ github.event_name }}", + EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + REF: "${{ github.ref }}", + WORKFLOW_REPOSITORY: "${{ github.repository }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", + }; + if ( + !isDeepStrictEqual( + credentialAuthorization.env, + expectedCredentialAuthorizationEnvironment, + ) + ) { + errors.push( + "Manual PR credential authorization must bind the workflow and checkout identities", + ); + } + const authorizationSource = String(credentialAuthorization.run ?? ""); + for (const fragment of [ + '"$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw"', + '"$CHECKOUT_REPOSITORY" == "$WORKFLOW_REPOSITORY"', + '"$EVENT_NAME" == "workflow_dispatch"', + '"$REF" == "refs/heads/main"', + '"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', + '"$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$', + '"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA"', + '"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', + "credentials_allowed=false", + "credentials_allowed=true", + 'printf \'allowed=%s\\n\' "$credentials_allowed" >> "$GITHUB_OUTPUT"', + ]) { + if (!authorizationSource.includes(fragment)) { + errors.push(`Manual PR credential authorization must retain ${fragment}`); + } + } + for (const [jobName, job] of Object.entries(workflow.jobs)) { for (const step of job.steps ?? []) { const trustedHermesFixtureCheckout = diff --git a/tools/e2e/standard-profile-workflow-boundary.mts b/tools/e2e/standard-profile-workflow-boundary.mts index e4a2622dd66..22a5b5d0bb4 100644 --- a/tools/e2e/standard-profile-workflow-boundary.mts +++ b/tools/e2e/standard-profile-workflow-boundary.mts @@ -175,7 +175,7 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi shard: "${{ matrix.shard }}", artifact_layout: "${{ matrix.artifact_layout }}", trusted_main: - "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}", + "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}", })) { if (withInputs[name] !== expected) { errors.push(`${contract.job} must pass ${name} from the catalogue matrix`); @@ -388,7 +388,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi checkoutWith["persist-credentials"] !== false || workflowSteps.indexOf(checkout ?? {}) !== 2 ) { - errors.push("standard E2E profile must check out the exact candidate without credentials"); + errors.push("standard E2E profile must check out checkout_sha without credentials"); } const auth = requireStep(errors, workflowSteps, "Authenticate to Docker Hub");