Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions .github/actions/ci-reviewed-npm-audit/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,28 +34,36 @@ runs:
set -euo pipefail
current_bucket="$(( $(date -u +%s) / 43200 ))"
previous_bucket="$(( current_bucket - 1 ))"
input_digest="$(node --input-type=module - '${{ inputs.target-root }}' "$GITHUB_ACTION_PATH/../../../ci/reviewed-npm-audit.json" <<'NODE'
node --input-type=module - '${{ inputs.target-root }}' "$GITHUB_ACTION_PATH/../../../ci/reviewed-npm-audit.json" <<'NODE'
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { appendFileSync, readFileSync } from "node:fs";
import { join } from "node:path";
const [targetRoot, configFile] = process.argv.slice(2);
const configSource = readFileSync(configFile, "utf8");
const config = JSON.parse(configSource);
if (!/^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$/.test(config.npmVersion) || /[\r\n]/.test(config.npmVersion)) {
throw new Error("reviewed npm audit configuration has an invalid npmVersion");
}
if (!/^sha512-[A-Za-z0-9+/]+={0,2}$/.test(config.npmIntegrity) || /[\r\n]/.test(config.npmIntegrity)) {
throw new Error("reviewed npm audit configuration has an invalid npmIntegrity");
}
const directories = ["", ...config.lockedGraphs.map((graph) => graph.directory)].sort();
const hash = createHash("sha256");
hash.update(configSource);
hash.update(JSON.stringify({ argv: ["audit", "--registry=https://registry.yarnpkg.com", "--omit=dev", "--json"], npmVersion: "10.9.4", registry: "https://registry.yarnpkg.com/", schemaVersion: 1 }));
hash.update(JSON.stringify({ argv: ["audit", "--registry=https://registry.yarnpkg.com", "--omit=dev", "--json"], npmVersion: config.npmVersion, registry: "https://registry.yarnpkg.com/", schemaVersion: 1 }));
for (const directory of directories) {
for (const file of ["package.json", "package-lock.json"]) {
const relative = join(directory, file);
hash.update(relative);
hash.update(readFileSync(join(targetRoot, relative)));
}
}
process.stdout.write(hash.digest("hex"));
appendFileSync(
process.env.GITHUB_OUTPUT,
`input-digest=${hash.digest("hex")}\nnpm-version=${config.npmVersion}\nnpm-integrity=${config.npmIntegrity}\n`,
);
NODE
)"
printf 'current=%s\nprevious=%s\ninput-digest=%s\n' "$current_bucket" "$previous_bucket" "$input_digest" >> "$GITHUB_OUTPUT"
printf 'current=%s\nprevious=%s\n' "$current_bucket" "$previous_bucket" >> "$GITHUB_OUTPUT"
mkdir -p "${{ inputs.cache-directory }}"

- name: Restore current reviewed npm audit cache bucket
Expand All @@ -75,9 +83,8 @@ runs:
- name: Download and verify production npm
shell: bash
env:
NEMOCLAW_REVIEWED_NPM_VERSION: "10.9.4"
NEMOCLAW_REVIEWED_NPM_INTEGRITY: >-
sha512-OnUG836FwboQIbqtefDNlyR0gTHzIfwRfE3DuiNewBvnMnWEpB0VEXwBlFVgqpNzIgYo/MHh3d2Hel/pszapAA==
NEMOCLAW_REVIEWED_NPM_VERSION: ${{ steps.cache-buckets.outputs.npm-version }}
NEMOCLAW_REVIEWED_NPM_INTEGRITY: ${{ steps.cache-buckets.outputs.npm-integrity }}
run: env -u NODE_AUTH_TOKEN -u NPM_TOKEN -u NPM_CONFIG__AUTH_TOKEN "$GITHUB_ACTION_PATH/verify-and-install-npm.sh"

- name: Materialize and audit reviewed npm graphs
Expand Down
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ COPY agents/openclaw/mcporter-runtime/package.json /usr/local/lib/nemoclaw/mcpor
COPY agents/openclaw/mcporter-runtime/package-lock.json /usr/local/lib/nemoclaw/mcporter-runtime/package-lock.json
COPY agents/openclaw/wechat-runtime/package.json /usr/local/lib/nemoclaw/wechat-runtime/package.json
COPY agents/openclaw/wechat-runtime/package-lock.json /usr/local/lib/nemoclaw/wechat-runtime/package-lock.json
COPY ci/npm-audit-exceptions.json /scripts/npm-audit-exceptions.json
COPY ci/*npm-audit*.json /scripts/
COPY scripts/lib/reviewed-npm-archive.mts /scripts/lib/reviewed-npm-archive.mts
COPY scripts/lib/bundled-npm-package.mts /scripts/lib/bundled-npm-package.mts
COPY scripts/lib/reviewed-npm-audit.mts /scripts/lib/reviewed-npm-audit.mts
Expand Down Expand Up @@ -997,7 +997,7 @@ node --experimental-strip-types /scripts/lib/npm-audit-receipt.mts \
--package-json /usr/local/lib/nemoclaw/mcporter-runtime/package.json \
--package-lock /usr/local/lib/nemoclaw/mcporter-runtime/package-lock.json \
--raw-report "$MCPORTER_RAW_REPORT" --exceptions /scripts/npm-audit-exceptions.json \
--graph mcporter-runtime --npm-version "$(npm --version)" \
--graph mcporter-runtime --audit-config /scripts/reviewed-npm-audit.json \
--registry https://registry.yarnpkg.com --threshold high --legacy-npmjs true; \
else \
node --experimental-strip-types /scripts/lib/reviewed-npm-audit.mts \
Expand Down
8 changes: 4 additions & 4 deletions Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ COPY agents/openclaw/openclaw-runtime/package.json \
COPY agents/openclaw/mcporter-runtime/package.json \
agents/openclaw/mcporter-runtime/package-lock.json \
/usr/local/lib/nemoclaw/mcporter-runtime/
COPY ci/npm-audit-exceptions.json /scripts/npm-audit-exceptions.json
COPY ci/*npm-audit*.json /scripts/
COPY scripts/lib/reviewed-npm-archive.mts \
scripts/lib/bundled-npm-package.mts \
scripts/lib/reviewed-npm-audit.mts \
Expand Down Expand Up @@ -593,11 +593,11 @@ RUN --mount=type=bind,source=nemoclaw-blueprint/blueprint.yaml,target=/tmp/bluep
--package-json /usr/local/lib/nemoclaw/mcporter-runtime/package.json \
--package-lock /usr/local/lib/nemoclaw/mcporter-runtime/package-lock.json \
--raw-report "$MCPORTER_RAW_REPORT" --exceptions /scripts/npm-audit-exceptions.json \
--graph mcporter-runtime --npm-version "$(npm --version)" \
--graph mcporter-runtime --audit-config /scripts/reviewed-npm-audit.json \
--registry https://registry.yarnpkg.com --threshold high \
--legacy-npmjs true \
--result /tmp/mcporter-npm-audit-policy.json; \
cp "$MCPORTER_RAW_REPORT" /tmp/mcporter-npm-audit.json; \
--result /tmp/mcporter-npm-audit-policy.json \
--raw-copy /tmp/mcporter-npm-audit.json; \
else \
node --experimental-strip-types /scripts/lib/reviewed-npm-audit.mts \
--directory /usr/local/lib/nemoclaw/mcporter-runtime \
Expand Down
2 changes: 2 additions & 0 deletions ci/reviewed-npm-audit.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
{
"schemaVersion": 2,
"nodeVersion": "22.23.2",
"npmVersion": "10.9.4",
"npmIntegrity": "sha512-OnUG836FwboQIbqtefDNlyR0gTHzIfwRfE3DuiNewBvnMnWEpB0VEXwBlFVgqpNzIgYo/MHh3d2Hel/pszapAA==",
"registryOrigin": "https://registry.npmjs.org/",
"sourceRegistryPackage": {
"artifactName": "nvidia-openshell-sdk-0.0.106.tgz",
Expand Down
13 changes: 13 additions & 0 deletions scripts/audit-reviewed-npm-graph.mts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@
exceptionFile: string;
lockedGraphs: readonly LockedGraph[];
nodeVersion: string;
npmIntegrity: string;
npmVersion: string;
registryOrigin: string;
schemaVersion: 2;
severityThreshold: Severity;
Expand Down Expand Up @@ -195,6 +197,12 @@
parsed.archiveTarVersion !== "7.5.21" ||
typeof parsed.exceptionFile !== "string" ||
!parsed.exceptionFile ||
typeof parsed.npmVersion !== "string" ||
!/^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$/.test(parsed.npmVersion) ||
/[\r\n]/.test(parsed.npmVersion) ||
typeof parsed.npmIntegrity !== "string" ||
!/^sha512-[A-Za-z0-9+/]+={0,2}$/.test(parsed.npmIntegrity) ||
/[\r\n]/.test(parsed.npmIntegrity) ||
typeof parsed.registryOrigin !== "string" ||
!parsed.registryOrigin ||
!Array.isArray(parsed.archivePackages) ||
Expand Down Expand Up @@ -259,7 +267,7 @@
) ||
new Set(parsed.lockedGraphs.map(({ id }) => id)).size !== parsed.lockedGraphs.length
) {
throw new Error("ci/reviewed-npm-audit.json is invalid");

Check failure on line 270 in scripts/audit-reviewed-npm-graph.mts

View workflow job for this annotation

GitHub Actions / cli-test-shards (7)

[integration] test/repository/prepare-ci-npm-install.test.ts > trusted OpenShell SDK archive preparation > installs the reviewed archive offline after npm stages it

Error: ci/reviewed-npm-audit.json is invalid ❯ parseAuditConfig scripts/audit-reviewed-npm-graph.mts:270:11 ❯ prepareCiNpmInstallWithReviewedConfig scripts/checks/prepare-ci-npm-install.mts:274:49 ❯ test/repository/prepare-ci-npm-install.test.ts:303:11

Check failure on line 270 in scripts/audit-reviewed-npm-graph.mts

View workflow job for this annotation

GitHub Actions / cli-test-shards (7)

[integration] test/repository/prepare-ci-npm-install.test.ts > trusted OpenShell SDK archive preparation > uses registry mode without requiring or caching an archive

Error: ci/reviewed-npm-audit.json is invalid ❯ parseAuditConfig scripts/audit-reviewed-npm-graph.mts:270:11 ❯ prepareCiNpmInstallWithReviewedConfig scripts/checks/prepare-ci-npm-install.mts:274:49 ❯ test/repository/prepare-ci-npm-install.test.ts:277:11

Check failure on line 270 in scripts/audit-reviewed-npm-graph.mts

View workflow job for this annotation

GitHub Actions / cli-test-shards (7)

[integration] test/repository/prepare-ci-npm-install.test.ts > trusted OpenShell SDK archive preparation > passes the verified archive from the plugin lock to npm cache preparation

Error: ci/reviewed-npm-audit.json is invalid ❯ parseAuditConfig scripts/audit-reviewed-npm-graph.mts:270:11 ❯ prepareCiNpmInstallWithReviewedConfig scripts/checks/prepare-ci-npm-install.mts:274:49 ❯ test/repository/prepare-ci-npm-install.test.ts:263:11

Check failure on line 270 in scripts/audit-reviewed-npm-graph.mts

View workflow job for this annotation

GitHub Actions / cli-test-shards (7)

[integration] test/repository/prepare-ci-npm-install.test.ts > trusted OpenShell SDK archive preparation > passes the verified archive from the root lock to npm cache preparation

Error: ci/reviewed-npm-audit.json is invalid ❯ parseAuditConfig scripts/audit-reviewed-npm-graph.mts:270:11 ❯ prepareCiNpmInstallWithReviewedConfig scripts/checks/prepare-ci-npm-install.mts:274:49 ❯ test/repository/prepare-ci-npm-install.test.ts:249:11
}
return parsed;
}
Expand Down Expand Up @@ -909,6 +917,11 @@
fs.rmSync(artifactDirectory, { recursive: true, force: true });
fs.mkdirSync(artifactDirectory, { recursive: true });
const npmVersion = run("npm", ["--version"], TRUSTED_REPO_ROOT).stdout.trim();
if (npmVersion !== config.npmVersion) {
throw new Error(
`reviewed npm audit requires npm ${config.npmVersion}; running npm ${npmVersion}`,
);
}
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-reviewed-npm-audit-"));
try {
const sourceResult = auditSourceGraph(
Expand Down
38 changes: 33 additions & 5 deletions scripts/lib/npm-audit-receipt.mts
Original file line number Diff line number Diff line change
Expand Up @@ -211,13 +211,34 @@ export function canonicalAuditReceipt(receipt: AuditReceipt): string {
return `${JSON.stringify(receipt, null, 2)}\n`;
}

export function reviewedNpmVersionFromConfig(contents: string): string {
let parsed: unknown;
try {
parsed = JSON.parse(contents);
} catch {
throw new Error("reviewed npm audit configuration is not valid JSON");
}
const npmVersion =
typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
? (parsed as Record<string, unknown>).npmVersion
: undefined;
if (
typeof npmVersion !== "string" ||
!/^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$/.test(npmVersion) ||
/[\r\n]/.test(npmVersion)
) {
throw new Error("reviewed npm audit configuration has an invalid npmVersion");
}
return npmVersion;
}

function cli(args: readonly string[]): void {
const values = new Map<string, string>();
for (let index = 0; index < args.length; index += 2) {
const value = args[index + 1];
if (!args[index]?.startsWith("--") || value === undefined)
throw new Error(
"usage: npm-audit-receipt.mts --receipt FILE --package-json FILE --package-lock FILE --raw-report FILE --exceptions FILE --graph ID --npm-version VERSION --registry ORIGIN --threshold SEVERITY [--legacy-npmjs true]",
"usage: npm-audit-receipt.mts --receipt FILE --package-json FILE --package-lock FILE --raw-report FILE --exceptions FILE --graph ID --audit-config FILE --registry ORIGIN --threshold SEVERITY [--legacy-npmjs true] [--result FILE] [--raw-copy FILE]",
);
values.set(args[index], value);
}
Expand All @@ -228,14 +249,16 @@ function cli(args: readonly string[]): void {
"--raw-report",
"--exceptions",
"--graph",
"--npm-version",
"--audit-config",
"--registry",
"--threshold",
];
const allowed = [...required, "--result", "--legacy-npmjs"];
const allowed = [...required, "--result", "--raw-copy", "--legacy-npmjs"];
exactKeys(
Object.fromEntries(
[...values].filter(([key]) => key !== "--result" && key !== "--legacy-npmjs"),
[...values].filter(
([key]) => key !== "--result" && key !== "--raw-copy" && key !== "--legacy-npmjs",
),
),
required,
"verifier arguments",
Expand All @@ -246,9 +269,12 @@ function cli(args: readonly string[]): void {
const packageLock = fs.readFileSync(values.get("--package-lock")!);
const rawResponse = fs.readFileSync(values.get("--raw-report")!);
const exceptionPolicy = fs.readFileSync(values.get("--exceptions")!);
const npmVersion = reviewedNpmVersionFromConfig(
fs.readFileSync(values.get("--audit-config")!, "utf8"),
);
parseAndVerifyAuditReceipt(fs.readFileSync(values.get("--receipt")!, "utf8"), {
graphId: values.get("--graph")!,
npmVersion: values.get("--npm-version")!,
npmVersion,
exceptionPolicy,
severityThreshold: values.get("--threshold")! as AuditReceipt["severityThreshold"],
packageJson,
Expand All @@ -271,6 +297,8 @@ function cli(args: readonly string[]): void {
);
if (values.has("--result"))
fs.writeFileSync(values.get("--result")!, `${JSON.stringify(policyResult, null, 2)}\n`);
if (values.has("--raw-copy"))
fs.copyFileSync(values.get("--raw-report")!, values.get("--raw-copy")!);
console.log("reviewed npm audit receipt and current policy verified");
}

Expand Down
4 changes: 4 additions & 0 deletions src/lib/sandbox/build-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,10 @@ function stageOptimizedSandboxBuildContext(
path.join(rootDir, "ci", "npm-audit-exceptions.json"),
path.join(stagedCiDir, "npm-audit-exceptions.json"),
);
fs.copyFileSync(
path.join(rootDir, "ci", "reviewed-npm-audit.json"),
path.join(stagedCiDir, "reviewed-npm-audit.json"),
);
normalizeReadModesForDockerCopy(stagedCiDir);

fs.mkdirSync(stagedNemoclawDir, { recursive: true });
Expand Down
97 changes: 95 additions & 2 deletions test/automation/releases/npm-audit-receipt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ describe("reviewed npm audit receipt", () => {
fs.writeFileSync(path.join(root, "package-lock.json"), inputs.packageLock);
fs.writeFileSync(path.join(root, "exceptions.json"), inputs.exceptionPolicy);
fs.writeFileSync(path.join(root, "raw.json"), inputs.rawResponse);
fs.writeFileSync(
path.join(root, "reviewed-npm-audit.json"),
JSON.stringify({ npmVersion: inputs.npmVersion }),
);
const auditReceipt = legacy
? {
...receipt(new Date()),
Expand All @@ -194,8 +198,8 @@ describe("reviewed npm audit receipt", () => {
path.join(root, "exceptions.json"),
"--graph",
inputs.graphId,
"--npm-version",
inputs.npmVersion,
"--audit-config",
path.join(root, "reviewed-npm-audit.json"),
"--registry",
registry,
"--threshold",
Expand All @@ -216,4 +220,93 @@ describe("reviewed npm audit receipt", () => {
}
},
);

it.each([
{
error: "",
label: "the canonical npm version",
npmVersion: inputs.npmVersion,
policy: { graph: inputs.graphId, status: "clean" },
rawCopy: inputs.rawResponse,
status: 0,
},
{
error: "receipt identity does not match expected graph and npm",
label: "a mismatched npm version",
npmVersion: "11.18.0",
policy: undefined,
rawCopy: undefined,
status: 1,
},
])(
"executes the receipt producer-to-Docker-consumer contract for $label",
({ error, npmVersion, policy, rawCopy, status }) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "receipt-docker-contract-"));
const resultFile = path.join(root, "policy.json");
const rawCopyFile = path.join(root, "copied-raw.json");
try {
fs.writeFileSync(path.join(root, "package.json"), inputs.packageJson);
fs.writeFileSync(path.join(root, "package-lock.json"), inputs.packageLock);
fs.writeFileSync(path.join(root, "exceptions.json"), inputs.exceptionPolicy);
fs.writeFileSync(path.join(root, "raw.json"), inputs.rawResponse);
fs.writeFileSync(
path.join(root, "receipt.json"),
canonicalAuditReceipt(receipt(new Date())),
);
fs.writeFileSync(
path.join(root, "reviewed-npm-audit.json"),
JSON.stringify({ npmVersion }),
);

const result = spawnSync(
process.execPath,
[
"--experimental-strip-types",
path.join(import.meta.dirname, "../../../scripts/lib/npm-audit-receipt.mts"),
"--receipt",
path.join(root, "receipt.json"),
"--package-json",
path.join(root, "package.json"),
"--package-lock",
path.join(root, "package-lock.json"),
"--raw-report",
path.join(root, "raw.json"),
"--exceptions",
path.join(root, "exceptions.json"),
"--graph",
inputs.graphId,
"--audit-config",
path.join(root, "reviewed-npm-audit.json"),
"--registry",
inputs.registryOrigin,
"--threshold",
inputs.severityThreshold,
"--result",
resultFile,
"--raw-copy",
rawCopyFile,
],
{ encoding: "utf8" },
);

expect(result.status, result.stderr).toBe(status);
expect(fs.existsSync(resultFile)).toBe(status === 0);
expect(fs.existsSync(rawCopyFile)).toBe(status === 0);
const observedPolicy = fs.existsSync(resultFile)
? (JSON.parse(fs.readFileSync(resultFile, "utf8")) as Record<string, unknown>)
: undefined;
expect(
observedPolicy === undefined
? undefined
: { graph: observedPolicy.graph, status: observedPolicy.status },
).toEqual(policy);
expect(fs.existsSync(rawCopyFile) ? fs.readFileSync(rawCopyFile, "utf8") : undefined).toBe(
rawCopy,
);
expect(result.stderr).toContain(error);
} finally {
fs.rmSync(root, { recursive: true, force: true });
}
},
);
});
Loading
Loading