Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/actions/publish-base-image-manifest/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,4 @@ runs:
name: managed-base-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.agent }}
path: ${{ runner.temp }}/managed-base-contract/contract.json
if-no-files-found: error
retention-days: 1
retention-days: 90
51 changes: 49 additions & 2 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ jobs:
base-image-publication:
runs-on: ubuntu-latest
timeout-minutes: 55
outputs:
dcode_base_contract: ${{ steps.validate_dcode_base.outputs.contract }}
dcode_base_ref: ${{ steps.validate_dcode_base.outputs.base_ref }}
permissions:
actions: read
contents: read
Expand All @@ -122,7 +125,7 @@ jobs:
required=1
;;
NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:controller)
required=0
required=1
;;
*)
echo "::error::base-image publication mode is not trusted" >&2
Expand All @@ -145,19 +148,41 @@ jobs:
with:
node-version: 22

- name: Verify applicable base-image publication
- id: publication
name: Verify applicable base-image publication
if: ${{ steps.publication_mode.outputs.required == '1' }}
env:
EXPECTED_SHA: ${{ github.sha }}
GITHUB_TOKEN: ${{ github.token }}
run: node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30

- name: Download immutable Deep Agents Code base contract
if: ${{ steps.publication_mode.outputs.required == '1' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
github-token: ${{ github.token }}
name: managed-base-${{ steps.publication.outputs.run_id }}-${{ steps.publication.outputs.run_attempt }}-langchain-deepagents-code
path: ${{ runner.temp }}/dcode-base-contract
repository: NVIDIA/NemoClaw
run-id: ${{ steps.publication.outputs.run_id }}

- id: validate_dcode_base
name: Validate immutable Deep Agents Code base
if: ${{ steps.publication_mode.outputs.required == '1' }}
env:
PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
run: node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract/contract.json"

generate-matrix:
needs: base-image-publication
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
cli_artifact_provenance: ${{ steps.record_cli_artifact.outputs.provenance }}
dcode_base_contract: ${{ needs.base-image-publication.outputs.dcode_base_contract }}
dcode_base_ref: ${{ needs.base-image-publication.outputs.dcode_base_ref }}
matrix: ${{ steps.matrix.outputs.matrix }}
test_matrix: ${{ steps.matrix.outputs.test_matrix }}
hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
Expand Down Expand Up @@ -770,6 +795,7 @@ jobs:
include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live
NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF: ${{ needs.generate-matrix.outputs.dcode_base_ref }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
Expand Down Expand Up @@ -823,6 +849,26 @@ jobs:
with:
build-cli: "false"

- name: Record immutable Deep Agents Code base evidence
if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}
env:
BASE_CONTRACT: ${{ needs.generate-matrix.outputs.dcode_base_contract }}
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
evidence_dir="${E2E_ARTIFACT_DIR}/${TARGET_ID}"
install -d -m 0700 "${evidence_dir}"
jq -e -n \
--arg candidateSha "${CANDIDATE_SHA}" \
--argjson baseContract "${BASE_CONTRACT}" \
'{contractVersion: 1, candidateSha: $candidateSha, base: $baseContract}
| select(
(.candidateSha | test("^[0-9a-f]{40}$"))
and (.base.reference | test("@sha256:[0-9a-f]{64}$"))
)' >"${evidence_dir}/dcode-base-image.json"

- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
Expand Down Expand Up @@ -942,6 +988,7 @@ jobs:
e2e-artifacts/live/${{ matrix.id }}/environment.result.json
e2e-artifacts/live/${{ matrix.id }}/onboarding.result.json
e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json
e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json
e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json
e2e-artifacts/live/risk-signal.json
e2e-artifacts/live/${{ matrix.id }}/actions/
Expand Down
5 changes: 5 additions & 0 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@
"test": "hermes-e2e: install.sh onboards Hermes and proves health plus live inference",
"category": "security"
},
{
"file": "test/e2e/support/base-image-publication-workflow-boundary.test.ts",
"test": "retains immutable base contracts for later qualification (#9049)",
"category": "security"
},
{
"file": "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts",
"test": "binds the cleanup action and helper content to the pinned commit",
Expand Down
108 changes: 93 additions & 15 deletions test/e2e/support/base-image-publication-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import os from "node:os";
import path from "node:path";

import { describe, expect, it } from "vitest";
import YAML from "yaml";

import {
type OperationsWorkflow,
Expand All @@ -24,7 +25,9 @@ type MutableStep = {
};

type MutableJob = Record<string, unknown> & {
env?: Record<string, unknown>;
needs?: unknown;
outputs?: Record<string, unknown>;
permissions?: Record<string, unknown>;
steps?: MutableStep[];
};
Expand Down Expand Up @@ -90,20 +93,44 @@ function runClassifier(environment: {
}

describe("base-image publication workflow boundary (#7372)", () => {
// source-shape-contract: security -- Immutable base contracts must outlive the qualification interval so later E2E cannot fall back to a mutable alias.
it("retains immutable base contracts for later qualification (#9049)", () => {
const action = YAML.parse(
fs.readFileSync(
path.join(process.cwd(), ".github/actions/publish-base-image-manifest/action.yaml"),
"utf8",
),
) as { runs: { steps: MutableStep[] } };
const upload = action.runs.steps.find(
(step) => step.name === "Upload managed base image contract",
);

expect(upload).toMatchObject({
uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a",
with: {
"if-no-files-found": "error",
"retention-days": 90,
},
});
});

it.each([
["push to main", "push", "", "1"],
["manual main", "workflow_dispatch", "", "1"],
["controller-selected PR", "workflow_dispatch", "a".repeat(40), "0"],
])("classifies %s without executing untrusted code (#7372)", (_case, eventName, checkoutSha, required) => {
expect(
runClassifier({
checkoutSha,
eventName,
ref: "refs/heads/main",
repository: "NVIDIA/NemoClaw",
}),
).toEqual({ output: `required=${required}\n`, status: 0 });
});
["controller-selected PR", "workflow_dispatch", "a".repeat(40), "1"],
])(
"classifies %s without executing untrusted code (#7372)",
(_case, eventName, checkoutSha, required) => {
expect(
runClassifier({
checkoutSha,
eventName,
ref: "refs/heads/main",
repository: "NVIDIA/NemoClaw",
}),
).toEqual({ output: `required=${required}\n`, status: 0 });
},
);

it.each([
["a fork", "push", "", "refs/heads/main", "attacker/NemoClaw"],
Expand All @@ -116,9 +143,12 @@ describe("base-image publication workflow boundary (#7372)", () => {
"refs/heads/main",
"NVIDIA/NemoClaw",
],
])("rejects %s instead of skipping the gate (#7372)", (_case, eventName, checkoutSha, ref, repository) => {
expect(runClassifier({ checkoutSha, eventName, ref, repository }).status).not.toBe(0);
});
])(
"rejects %s instead of skipping the gate (#7372)",
(_case, eventName, checkoutSha, ref, repository) => {
expect(runClassifier({ checkoutSha, eventName, ref, repository }).status).not.toBe(0);
},
);

const mutations: Array<[string, (value: MutableWorkflow) => void]> = [
["runner size", (value) => (value.jobs["base-image-publication"]["runs-on"] = "self-hosted")],
Expand All @@ -142,7 +172,7 @@ describe("base-image publication workflow boundary (#7372)", () => {
[
"classifier outcome",
(value) => {
gateSteps(value)[0].run = gateSteps(value)[0].run!.replace("required=0", "required=1");
gateSteps(value)[0].run = gateSteps(value)[0].run!.replace("required=1", "required=0");
},
],
["checkout condition", (value) => (gateSteps(value)[1].if = "${{ always() }}")],
Expand All @@ -165,8 +195,56 @@ describe("base-image publication workflow boundary (#7372)", () => {
gateSteps(value)[3].run = "node tools/e2e/base-image-publication.mts";
},
],
[
"contract download pin",
(value) => (gateSteps(value)[4].uses = "actions/download-artifact@v8"),
],
[
"contract run binding",
(value) => (gateSteps(value)[4].with!["run-id"] = "${{ github.run_id }}"),
],
[
"contract validation",
(value) =>
(gateSteps(value)[5].run = "node tools/e2e/dcode-base-image-contract.mts contract.json"),
],
["step count", (value) => gateSteps(value).push({ name: "Unreviewed step", run: "true" })],
["fanout dependency", (value) => (value.jobs["generate-matrix"].needs = [])],
[
"matrix base output",
(value) => {
(value.jobs["generate-matrix"].outputs as Record<string, unknown>).dcode_base_ref =
"${{ inputs.base_ref }}";
},
],
[
"live mutable base",
(value) => {
value.jobs.live.env!.NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF =
"ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest";
},
],
[
"live base evidence ordering",
(value) => {
const steps = value.jobs.live.steps!;
const evidence = steps.find(
(step) => step.name === "Record immutable Deep Agents Code base evidence",
)!;
steps.splice(steps.indexOf(evidence), 1);
steps.push(evidence);
},
],
[
"live base evidence upload",
(value) => {
const upload = value.jobs.live.steps!.find((step) => step.name === "Upload E2E artifacts")!;
upload.with!.path = String(upload.with!.path).replace(
"e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json\n",
"",
);
},
],
];

it.each(mutations)("rejects %s drift (#7372)", (_case, mutate) => {
Expand Down
46 changes: 31 additions & 15 deletions test/e2e/support/base-image-publication.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
validatePublisherJobs,
validateWorkflow,
waitForBaseImagePublication,
writePublicationRunOutputs,
} from "../../../tools/e2e/base-image-publication.mts";

const EXPECTED_SHA = "a".repeat(40);
Expand Down Expand Up @@ -166,13 +167,12 @@ describe("base-image publication evidence", () => {
expect(isBaseImagePublicationEvent(eventName)).toBe(true);
});

it.each([
"schedule",
"pull_request",
undefined,
])("rejects unsupported %s publication preflight events", (eventName) => {
expect(isBaseImagePublicationEvent(eventName)).toBe(false);
});
it.each(["schedule", "pull_request", undefined])(
"rejects unsupported %s publication preflight events",
(eventName) => {
expect(isBaseImagePublicationEvent(eventName)).toBe(false);
},
);

it("extracts literal paths and the reviewed managed-image input families (#7372)", () => {
const source = fs.readFileSync(
Expand Down Expand Up @@ -474,14 +474,14 @@ describe("base-image publication evidence", () => {
).toMatchObject({ state: "pending", run: { status: "in_progress" } });
});

it.each([
"failure",
"cancelled",
] as const)("fails closed when publication concludes %s (#7372)", (conclusion) => {
expect(() =>
selectPublicationRun(runsPayload([workflowRun({ conclusion })]), history(), WORKFLOW_ID),
).toThrow(`base-image workflow for ${RELEVANT_SHA} concluded ${conclusion}; ${RUN_URL}`);
});
it.each(["failure", "cancelled"] as const)(
"fails closed when publication concludes %s (#7372)",
(conclusion) => {
expect(() =>
selectPublicationRun(runsPayload([workflowRun({ conclusion })]), history(), WORKFLOW_ID),
).toThrow(`base-image workflow for ${RELEVANT_SHA} concluded ${conclusion}; ${RUN_URL}`);
},
);

it("fails closed on ambiguous or malformed runs (#7372)", () => {
expect(() =>
Expand Down Expand Up @@ -538,6 +538,22 @@ describe("base-image publication evidence", () => {
).toThrow(/changed while evidence was verified/u);
});

it("exports the selected immutable publication identity for downstream qualification (#9049)", () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-publication-output-"));
const output = path.join(directory, "github-output");
try {
writePublicationRunOutputs(output, selectedRun());
expect(fs.readFileSync(output, "utf8")).toBe(
`run_id=${RUN_ID}\nrun_attempt=1\nhead_sha=${RELEVANT_SHA}\n`,
);
expect(() => writePublicationRunOutputs("bad\npath", selectedRun())).toThrow(
/single-line path/u,
);
} finally {
fs.rmSync(directory, { recursive: true, force: true });
}
});

it.each([
["missing", successfulJobs().slice(0, 2), /missing required/u],
[
Expand Down
Loading
Loading