From 5ea601f368b19de4e26bbb282a8ba6cfc0b85f96 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 28 Aug 2026 21:49:39 -0700 Subject: [PATCH 01/28] feat(openshell): add external gateway status Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 17 +- ci/source-architecture-budget.json | 2 +- ...ypescript-sdk-0.0.106-dependency-review.md | 154 ++++++ .../blueprint/runner-external-target.test.ts | 86 +++- nemoclaw/src/blueprint/runner.test.ts | 8 +- nemoclaw/src/blueprint/runner.ts | 100 +++- .../openshell-external-target-boundary.cts | 60 ++- ...openshell-external-target-boundary.test.ts | 19 +- .../openshell-gateway-health-sdk.test.ts | 200 ++++++++ .../shared/openshell-gateway-health-sdk.ts | 207 ++++++++ .../shared/openshell-observation-boundary.cts | 128 +++++ .../openshell-observation-boundary.test.ts | 129 +++++ nemoclaw/tsconfig.shared.json | 1 + nemoclaw/vitest.project.ts | 8 + package-lock.json | 53 +++ package.json | 9 +- .../lib/package-blueprint-runner-runtime.mts | 19 + .../adapters/openshell/sandbox-observer.ts | 24 + src/lib/blueprint-runner.ts | 45 ++ test/e2e/RETRY_INVENTORY.md | 8 + .../live/external-gateway-health-helpers.ts | 202 +++++--- test/e2e/live/external-gateway-health.test.ts | 4 +- test/e2e/mock-parity.json | 1 + .../cli-artifact-workflow-boundary.test.ts | 36 +- ...l-gateway-health-workflow-boundary.test.ts | 24 + .../blueprint-external-target-plan.test.ts | 440 ++++++++++++------ .../blueprint-external-target-timeout.test.ts | 305 ++++++++++++ .../cli/build-upgrade.test.ts | 9 + tools/e2e/cli-artifact-workflow-boundary.mts | 11 +- ...ernal-gateway-health-workflow-boundary.mts | 56 ++- vitest.config.ts | 7 + 31 files changed, 2112 insertions(+), 260 deletions(-) create mode 100644 internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md create mode 100644 nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts create mode 100644 nemoclaw/src/shared/openshell-gateway-health-sdk.ts create mode 100644 nemoclaw/src/shared/openshell-observation-boundary.cts create mode 100644 nemoclaw/src/shared/openshell-observation-boundary.test.ts create mode 100644 scripts/lib/package-blueprint-runner-runtime.mts create mode 100755 src/lib/blueprint-runner.ts create mode 100644 test/e2e/RETRY_INVENTORY.md create mode 100644 test/package-contract/blueprint-external-target-timeout.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index fb51b28309e..a80b3812e3a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -770,11 +770,18 @@ jobs: { echo "::error::checked-out commit does not match the artifact candidate SHA"; exit 1; } [[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ && "$RUN_ID" =~ ^[1-9][0-9]*$ ]] || { echo "::error::workflow run identity is invalid"; exit 1; } - for required_file in dist/nemoclaw.js dist/build-identity.json; do + for required_file in \ + dist/nemoclaw.js \ + dist/build-identity.json \ + dist/lib/blueprint-runner.js \ + dist/nemoclaw/package.json \ + dist/nemoclaw/blueprint/runner.js; do [[ -f "$required_file" && ! -L "$required_file" && -s "$required_file" ]] || { echo "::error::candidate CLI build output is missing or is not a nonempty regular file: $required_file"; exit 1; } done for boundary in \ + openshell-gateway-health-sdk.js \ + openshell-observation-boundary.cjs \ openshell-policy-boundary.cjs \ sandbox-name.cjs \ snapshot-sanitizer-boundary.cjs; do @@ -790,6 +797,12 @@ jobs: .sourceRevision == $candidateSha ' dist/build-identity.json >/dev/null || { echo "::error::candidate CLI build identity does not match the candidate commit SHA"; exit 1; } + jq -e ' + type == "object" and + (keys | sort) == ["type"] and + .type == "module" + ' dist/nemoclaw/package.json >/dev/null || + { echo "::error::candidate Blueprint Runner module boundary is invalid"; exit 1; } # BEGIN exact managed-image catalog staging managed_catalog="${RUNNER_TEMP}/pr-managed-image-catalog.json" @@ -3400,7 +3413,7 @@ jobs: E2E_DEFAULT_ENABLED: "0" E2E_TARGET_ID: "external-gateway-health" E2E_AGENT_RUNTIME: "none" - E2E_OBSERVABLE_OUTCOME: "The reviewed SDK observes exact public gateway health over explicit HTTPS and CA" + E2E_OBSERVABLE_OUTCOME: "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu host with OpenShell 0.0.106; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/external-gateway-health NEMOCLAW_RUN_LIVE_E2E: "1" diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 048f3dba5cd..b9d9feee87d 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -25,7 +25,7 @@ "src/lib/messaging/channels/index.ts": 25, "src/lib/onboard/gateway-binding.ts": 53, "src/lib/runner.ts": 85, - "src/lib/security/redact.ts": 54, + "src/lib/security/redact.ts": 55, "src/lib/state/onboard-session.ts": 36, "src/lib/state/mcp-lifecycle-lock.ts": 21, "src/lib/state/registry.ts": 100, diff --git a/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md b/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md new file mode 100644 index 00000000000..6b4062eb0aa --- /dev/null +++ b/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md @@ -0,0 +1,154 @@ + + + +# OpenShell TypeScript SDK 0.0.106 dependency review + +> Internal engineering evidence. This file is not part of the public documentation set. + +## Status and scope + +Issue #9872 permits `@nvidia/openshell-sdk@0.0.106` for one bounded public gateway-health call. +The slice reads no machine credential and makes no authenticated or mutating call. + +This review applies only to the exact source, package, and runtime graph below. +It does not qualify a Runner image or approve machine authentication. + +## Source and package identity + +| Evidence | Reviewed value | +| --- | --- | +| OpenShell tag | `v0.0.106` | +| Source commit | `c4b500a7de64d0b66e3ee8098f58d14299092162` | +| Package | `@nvidia/openshell-sdk@0.0.106` | +| Registry | GitHub Packages | +| Package version ID | `1134230109` | +| Package SHA-512 integrity | `sha512-dB4mLex23Pnw61caGMR2CMHQihy9bj7IK2elJJd718k3yevm+fOt/vG6dJg8/5us4la2BwcOdRwLvOia3tdwFw==` | +| Observed package SHA-256 | `30ae4c4749a610cc6e9b22067ece61f6137ba858acf5102c23cb81c37c4cc3fb` | +| Producer run | `31809093046`, attempt 2, job `94864069197` | + +The registry metadata and the OpenShell tag identify the same source commit. +The package has no registry signature or provenance attestation. +The root package lock binds the exact registry URL and SHA-512 integrity. +A later SDK version requires a new dependency review. + +## Selected runtime graph + +| Package | Version | Declared license | +| --- | --- | --- | +| `@nvidia/openshell-sdk` | `0.0.106` | Apache-2.0 | +| `@bufbuild/protobuf` | `2.12.1` | Apache-2.0 AND BSD-3-Clause | +| `@connectrpc/connect` | `2.1.2` | Apache-2.0 | +| `@connectrpc/connect-node` | `2.1.2` | Apache-2.0 | + +The root manifest records all four packages as exact optional dependencies. +The package lock pins each version and package integrity for source installs. +None of these packages defines an install lifecycle script. + +## API and security contract + +NemoClaw uses `OpenShellClient.connect({ gateway, caCert })` from the official SDK. It calls the +SDK-documented `client.raw.health()` operation over the shared transport created and owned by that +client. NemoClaw does not import ConnectRPC, create a generated client, or create an HTTP/2 session. + +The health adapter: + +- Accepts only the validated HTTPS endpoint and CA. +- Gives the endpoint and a copy of the CA bytes to the official SDK without an authentication token, + client certificate, client key, or insecure TLS option. +- Retains the SDK's normal TLS certificate and hostname verification. +- Starts one five-second deadline before local SDK loading and bounds adapter return through SDK + client creation and public health. +- Passes that same deadline signal to the SDK health operation. +- Passes no token, client certificate, client key, CLI state, or insecure TLS option. +- Replaces SDK import and transport details with fixed messages. +- Uses no direct transport or fallback. + +The SDK does not expose a DNS lookup hook or transport close handle. It uses the platform resolver +when the explicit endpoint contains a hostname. The configured CA and TLS hostname check still +authenticate the remote endpoint, but NemoClaw cannot pin one DNS answer set through this SDK API. +Under the accepted initial contract, cluster and storage administrators are trusted infrastructure +actors, and this operation sends only unauthenticated public health. A later change that sends a +credential or accepts a broader trust model must revisit DNS rebinding and transport ownership. + +The SDK client is lazy. +Public health is the first network request. +Identity, workspace inventory, provider inventory, authentication, and mutation remain outside this slice. + +## Installation and redistribution + +The root NemoClaw package records the SDK and its three runtime dependencies as exact optional dependencies. +Installations that do not use external gateway status can omit private-registry access. +The root package exposes the Blueprint Runner executable. +The OpenClaw plugin package and lock do not contain the SDK. +The reviewed package workflow from PR #10368 supplies exact SDK bytes to untrusted CI without package credentials. + +If the runtime omits the SDK or exposes an incompatible client, external gateway status returns a fixed package-unavailable error before a network request. +The result exposes no import detail and starts no fallback. + +The SDK and its three runtime dependency archives omit license and notice files. +Before a Runner image distributes this graph, the image release must include the applicable Apache-2.0 and BSD-3-Clause license texts. +The image software bill of materials and third-party notice must bind to the installed graph. + +## Concern records + +### DEP-1: Package substitution + +- Severity and confidence: high, high confidence. +- Failure mode: another package or transitive version executes in the Runner. +- Control: pin the SDK and each runtime dependency exactly in the published root manifest, and pin each package URL and integrity in the source lock. +- Verification: compiled package installation and Runner package-contract tests. + +### DEP-2: Package credential exposure + +- Severity and confidence: high, high confidence. +- Failure mode: candidate code or an unreviewed dependency receives the package credential. +- Control: use the reviewed base-controlled package workflow from PR #10368. +- Verification: workflow and integration tests owned by that merged change. + +### DEP-3: Optional dependency absence or initialization failure + +- Severity and confidence: medium, high confidence. +- Failure mode: package failure looks like a gateway outage, or import detail reaches output. +- Control: validate the official SDK client export and generated service-status enum before network + access, then return a fixed package-unavailable error. +- Verification: focused adapter tests and the compiled package contract. + +### DEP-4: DNS rebinding or local-target substitution + +- Severity and confidence: high, high confidence. +- Failure mode: a validated hostname resolves or rebinds to a local service when the request starts. +- Control: the official SDK retains TLS certificate and hostname verification against the explicit + CA, and the slice sends only unauthenticated public health. The accepted initial contract treats + platform infrastructure administrators as trusted. +- Residual: SDK 0.0.106 does not expose a pinned lookup. A hostname can resolve differently between + validation and connection, including to a local address. TLS identity verification prevents an + unrelated local service from impersonating the configured gateway, but it does not prevent the + connection attempt. +- Verification: wrong-CA and wrong-hostname package tests; independent security review must keep + this residual visible. + +### DEP-5: Transport outlives the bounded observation + +- Severity and confidence: high, high confidence. +- Failure mode: request cancellation returns, but the HTTP/2 session keeps the Runner process alive. +- Control: pass the one total deadline signal to the official SDK health operation and retain the + bounded Kubernetes Job deadline as the outer control. A late local SDK import cannot start a + network request because health begins only after the bounded load returns. +- Residual: SDK 0.0.106 exposes no transport close handle. +- Verification: adapter deadline tests and a packaged Runner test against a stalled TLS peer prove + that the Runner returns and the process exits within the package boundary. + +### DEP-6: Redistribution evidence + +- Severity and confidence: medium, high confidence. +- Failure mode: a Runner image omits license evidence or claims unavailable provenance. +- Control: bind the image software bill of materials and license inventory to the installed graph. +- Verification: future Runner image release evidence. + +## Remaining gates + +- The feature must pass focused, package, type, build, and repository checks on its final commit. +- The final security review must explicitly assess the SDK's standard DNS resolution and lack of a + transport close handle; this review does not claim address pinning or session ownership. +- The future Runner image must prove the exact SDK runtime identity, software bill of materials, and license inventory. +- A separate accepted decision must define machine authentication before credential reads or authenticated calls start. diff --git a/nemoclaw/src/blueprint/runner-external-target.test.ts b/nemoclaw/src/blueprint/runner-external-target.test.ts index 5d2445c24bd..efdd31c11ba 100644 --- a/nemoclaw/src/blueprint/runner-external-target.test.ts +++ b/nemoclaw/src/blueprint/runner-external-target.test.ts @@ -17,7 +17,10 @@ const stdoutCapture = createStdoutCapture(); const mockExeca = vi.fn(); const externalTargetBoundaryMocks = vi.hoisted(() => ({ buildSanitizedExternalOpenShellTargetPlan: vi.fn(), + withExternalOpenShellTargetCa: vi.fn(), })); +const observeHealth = vi.fn(); +const gatewayHealthObserver = { observeHealth }; vi.mock("node:os", () => ({ homedir: () => FAKE_HOME })); @@ -87,6 +90,10 @@ function seedExternalTarget(): void { addFile("blueprint.yaml", YAML.stringify(externalTargetBlueprint())); } +function runMain(argv: string[]): Promise { + return main(argv, { gatewayHealthObserver }); +} + describe("Blueprint Runner external OpenShell target", () => { beforeEach(() => { store.clear(); @@ -96,6 +103,14 @@ describe("Blueprint Runner external OpenShell target", () => { externalTargetBoundaryMocks.buildSanitizedExternalOpenShellTargetPlan.mockReturnValue( SANITIZED_TARGET_PLAN, ); + externalTargetBoundaryMocks.withExternalOpenShellTargetCa.mockImplementation( + async (_target, _compatibility, useTarget) => + useTarget(SANITIZED_TARGET_PLAN, Buffer.from("public-ca-certificate")), + ); + observeHealth.mockResolvedValue({ + ok: true, + value: { status: "healthy", release: "0.0.106" }, + }); delete process.env.NEMOCLAW_BLUEPRINT_PATH; vi.spyOn(process.stdout, "write").mockImplementation(stdoutCapture.write); }); @@ -109,7 +124,7 @@ describe("Blueprint Runner external OpenShell target", () => { vi.stubEnv("NEMOCLAW_GATEWAY_MANAGEMENT", "/private/ambient-gateway-management.json"); seedExternalTarget(); - await main(["plan", "--dry-run"]); + await runMain(["plan", "--dry-run"]); expect(stdoutCapture.jsonOutput()).toEqual({ run_id: expect.stringMatching(/^nc-/), @@ -136,7 +151,7 @@ describe("Blueprint Runner external OpenShell target", () => { it("rejects apply before any subprocess or run-state effect (#9872)", async () => { seedExternalTarget(); - await expect(main(["apply"])).rejects.toThrow( + await expect(runMain(["apply"])).rejects.toThrow( /External OpenShell target apply is not available/, ); @@ -146,11 +161,70 @@ describe("Blueprint Runner external OpenShell target", () => { expect([...store.keys()].join("\n")).not.toContain("plan.json"); }); + it("reports explicit public gateway health without ambient or authentication access (#9872)", async () => { + vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://ambient-gateway.invalid"); + vi.stubEnv("NEMOCLAW_GATEWAY_MANAGEMENT", "/private/ambient-gateway-management.json"); + seedExternalTarget(); + + await runMain(["status", "--external-target"]); + + expect(stdoutCapture.jsonOutput()).toEqual({ + run_id: expect.stringMatching(/^nc-/), + openshell_target: SANITIZED_TARGET_PLAN, + gateway: { status: "healthy", release: "0.0.106" }, + compatibility: "compatible", + }); + expect(externalTargetBoundaryMocks.withExternalOpenShellTargetCa).toHaveBeenCalledWith( + externalTargetBlueprint().openshell_target as Record, + { minVersion: "0.0.106", maxVersion: "0.0.106" }, + expect.any(Function), + ); + expect(observeHealth).toHaveBeenCalledWith({ + target: SANITIZED_TARGET_PLAN, + caBundle: Uint8Array.from(Buffer.from("public-ca-certificate")), + timeoutMs: 5_000, + }); + expect(mockExeca).not.toHaveBeenCalled(); + expect(mockedValidateEndpoint).not.toHaveBeenCalled(); + const output = stdoutCapture.text(); + expect(output).not.toContain(EXTERNAL_CA_FILE); + expect(output).not.toContain(EXTERNAL_AUTHENTICATION_FILE); + expect(output).not.toContain("ambient-gateway"); + }); + + it("fails closed when the root OpenShell observer is not injected (#9872)", async () => { + seedExternalTarget(); + + await expect(main(["status", "--external-target"])).rejects.toThrow( + "The external OpenShell gateway observer is unavailable.", + ); + + expect(observeHealth).not.toHaveBeenCalled(); + expect(externalTargetBoundaryMocks.withExternalOpenShellTargetCa).not.toHaveBeenCalled(); + expect(mockExeca).not.toHaveBeenCalled(); + }); + + it.each([ + ["run receipt", ["status", "--external-target", "--run-id", "existing"]], + ["managed profile", ["status", "--external-target", "--profile", "default"]], + ["another action", ["plan", "--external-target"]], + ])( + "rejects external status with %s options before the health call (#9872)", + async (_name, argv) => { + seedExternalTarget(); + + await expect(runMain(argv)).rejects.toThrow(); + + expect(observeHealth).not.toHaveBeenCalled(); + expect(mockExeca).not.toHaveBeenCalled(); + }, + ); + it("rejects an inference endpoint override before any effect (#9872)", async () => { seedExternalTarget(); await expect( - main(["plan", "--endpoint-url", "https://override.example.test/v1"]), + runMain(["plan", "--endpoint-url", "https://override.example.test/v1"]), ).rejects.toThrow(/--endpoint-url configures inference/); expect(mockExeca).not.toHaveBeenCalled(); @@ -167,7 +241,7 @@ describe("Blueprint Runner external OpenShell target", () => { async (field, value) => { addFile("blueprint.yaml", YAML.stringify({ ...externalTargetBlueprint(), ...value })); - await expect(main(["plan"])).rejects.toThrow( + await expect(runMain(["plan"])).rejects.toThrow( `External OpenShell target planning does not accept '${field}'.`, ); @@ -183,7 +257,7 @@ describe("Blueprint Runner external OpenShell target", () => { it("rejects an explicit inference profile before any effect (#9872)", async () => { seedExternalTarget(); - await expect(main(["plan", "--profile", "default"])).rejects.toThrow( + await expect(runMain(["plan", "--profile", "default"])).rejects.toThrow( "--profile configures managed inference and is not accepted by external target-only planning.", ); @@ -208,7 +282,7 @@ describe("Blueprint Runner external OpenShell target", () => { }, ); - await expect(main(["plan"])).rejects.toThrow( + await expect(runMain(["plan"])).rejects.toThrow( "external OpenShell target endpoint port must be between 1 and 65535", ); diff --git a/nemoclaw/src/blueprint/runner.test.ts b/nemoclaw/src/blueprint/runner.test.ts index d43b8f7b9ee..9b773821008 100644 --- a/nemoclaw/src/blueprint/runner.test.ts +++ b/nemoclaw/src/blueprint/runner.test.ts @@ -1432,14 +1432,14 @@ describe("runner", () => { seedBlueprintFile(); }); - it("throws on unknown action with the raw invalid token", async () => { + it("throws a fixed diagnostic on an unknown action", async () => { store.clear(); - await expect(main(["bogus"])).rejects.toThrow(/Unknown action 'bogus'/); + await expect(main(["bogus"])).rejects.toThrow(/Unknown action\. Use:/); }); - it("throws on missing action with a clear marker", async () => { + it("throws on missing action", async () => { store.clear(); - await expect(main([])).rejects.toThrow(/Unknown action '\(missing\)'/); + await expect(main([])).rejects.toThrow(/Unknown action\. Use:/); }); it("parses plan with --profile and --dry-run", async () => { diff --git a/nemoclaw/src/blueprint/runner.ts b/nemoclaw/src/blueprint/runner.ts index eace7de99cd..9d147de4a49 100644 --- a/nemoclaw/src/blueprint/runner.ts +++ b/nemoclaw/src/blueprint/runner.ts @@ -36,9 +36,14 @@ import { buildSubprocessEnv } from "../lib/subprocess-env.js"; import { isPlainObject, type UnknownRecord } from "../shared/object-record.js"; import * as importedOpenShellGatewayEndpointBoundary from "../shared/openshell-gateway-endpoint-boundary.cjs"; import * as importedOpenShellExternalTargetBoundary from "../shared/openshell-external-target-boundary.cjs"; +import * as importedOpenShellObservationBoundary from "../shared/openshell-observation-boundary.cjs"; import * as importedOpenShellPolicyBoundary from "../shared/openshell-policy-boundary.cjs"; import * as importedSandboxName from "../shared/sandbox-name.cjs"; import type { SanitizedExternalOpenShellTargetPlan } from "../shared/openshell-external-target-boundary.cjs"; +import type { + ExternalOpenShellGatewayStatus, + OpenShellGatewayHealthObserver, +} from "../shared/openshell-observation-boundary.cjs"; import { attachRuntimeIdentity, buildRuntimeIdentityPlan, @@ -89,10 +94,18 @@ const sourceOrGeneratedOpenShellExternalTargetBoundary = importedOpenShellExternalTargetBoundary as typeof importedOpenShellExternalTargetBoundary & { default?: typeof importedOpenShellExternalTargetBoundary; }; -const { buildSanitizedExternalOpenShellTargetPlan } = +const { buildSanitizedExternalOpenShellTargetPlan, withExternalOpenShellTargetCa } = sourceOrGeneratedOpenShellExternalTargetBoundary.default ?? sourceOrGeneratedOpenShellExternalTargetBoundary; +const sourceOrGeneratedOpenShellObservationBoundary = + importedOpenShellObservationBoundary as typeof importedOpenShellObservationBoundary & { + default?: typeof importedOpenShellObservationBoundary; + }; +const { observeExternalOpenShellGatewayHealth } = + sourceOrGeneratedOpenShellObservationBoundary.default ?? + sourceOrGeneratedOpenShellObservationBoundary; + // sourceOfTruth: nemoclaw/src/shared/sandbox-name.cts const sourceOrGeneratedSandboxName = importedSandboxName as typeof importedSandboxName & { default?: typeof importedSandboxName; @@ -102,6 +115,8 @@ const { assertValidName, assertValidProviderName, isValidName } = type Action = "plan" | "apply" | "status" | "reconcile" | "rollback"; +type ExternalOpenShellTargetStatus = ExternalOpenShellGatewayStatus & Readonly<{ run_id: string }>; + type RollbackPlanSource = { sandbox_name?: unknown; sandbox_created_by_apply?: unknown; @@ -710,12 +725,17 @@ export function loadBlueprint(): Blueprint { try { content = readFileSync(bpFile, "utf-8"); } catch { - throw new Error(`blueprint.yaml not found at ${bpFile}`); + throw new Error("blueprint.yaml not found"); + } + let parsed: unknown; + try { + parsed = YAML.parse(content); + } catch { + throw new Error("blueprint.yaml contains invalid YAML"); } - const parsed: unknown = YAML.parse(content); if (!isBlueprint(parsed)) { throw new Error( - `blueprint.yaml at ${bpFile} must contain a YAML mapping with valid nested component shapes`, + "blueprint.yaml must contain a YAML mapping with valid nested component shapes", ); } return parsed; @@ -1726,6 +1746,50 @@ export function actionExternalOpenShellTargetPlan( return plan; } +export async function actionExternalOpenShellTargetStatus( + blueprint: Blueprint, + observer: OpenShellGatewayHealthObserver, +): Promise { + if (blueprint.openshell_target === undefined) { + throw new Error("blueprint does not declare an external OpenShell target"); + } + if ( + blueprint.min_openshell_version === undefined || + blueprint.max_openshell_version === undefined + ) { + throw new Error( + "External OpenShell target status requires blueprint min_openshell_version and max_openshell_version.", + ); + } + for (const field of ["components", "profiles", "min_openclaw_version"] as const) { + if (blueprint[field] !== undefined) { + throw new Error(`External OpenShell target status does not accept '${field}'.`); + } + } + + const rid = emitRunId(); + progress(10, "Validating external OpenShell target"); + const observation = await withExternalOpenShellTargetCa( + blueprint.openshell_target, + { + minVersion: blueprint.min_openshell_version, + maxVersion: blueprint.max_openshell_version, + }, + async (target, caContents) => { + const observed = await observeExternalOpenShellGatewayHealth(observer, { + target, + caBundle: caContents, + timeoutMs: 5_000, + }); + if (!observed.ok) throw new Error(observed.error.message); + return Object.freeze({ run_id: rid, ...observed.value }); + }, + ); + progress(100, "External target status complete"); + log(JSON.stringify(observation, null, 2)); + return observation; +} + export async function actionApply( profile: string, blueprint: Blueprint, @@ -2637,6 +2701,7 @@ export async function actionRollback(rid: string): Promise { export async function main( argv: string[] = process.argv.slice(2), options: { + gatewayHealthObserver?: OpenShellGatewayHealthObserver; snapshotCommand?: SnapshotCommandOptions; /** Code-only conformance-test seam. No CLI flag or environment input populates this policy. */ runtimeIdentityProfilePolicy?: RuntimeIdentityProfilePolicy; @@ -2650,6 +2715,7 @@ export async function main( let runId: string | undefined; let dryRun = false; let endpointUrl: string | undefined; + let externalTargetStatus = false; function requireValue(flag: string, i: number): string { if (i >= argv.length) throw new Error(`${flag} requires a value`); @@ -2661,9 +2727,7 @@ export async function main( actionSnapshots(argv.slice(1), options.snapshotCommand); return; } - throw new Error( - `Unknown action '${rawAction ?? "(missing)"}'. Use: plan, apply, status, reconcile, rollback, snapshots`, - ); + throw new Error("Unknown action. Use: plan, apply, status, reconcile, rollback, snapshots"); } for (let i = 1; i < argv.length; i++) { @@ -2684,9 +2748,16 @@ export async function main( case "--endpoint-url": endpointUrl = requireValue("--endpoint-url", ++i); break; + case "--external-target": + externalTargetStatus = true; + break; } } + if (externalTargetStatus && action !== "status") { + throw new Error("--external-target is accepted only with status"); + } + switch (action) { case "plan": { const blueprint = loadBlueprint(); @@ -2717,7 +2788,20 @@ export async function main( break; } case "status": - actionStatus(runId); + if (externalTargetStatus) { + if (runId !== undefined) { + throw new Error("--external-target and --run-id cannot be used together"); + } + if (profileProvided || planPath !== undefined || dryRun || endpointUrl !== undefined) { + throw new Error("External target status does not accept managed-run options"); + } + if (options.gatewayHealthObserver === undefined) { + throw new Error("The external OpenShell gateway observer is unavailable."); + } + await actionExternalOpenShellTargetStatus(loadBlueprint(), options.gatewayHealthObserver); + } else { + actionStatus(runId); + } break; case "reconcile": if (!runId) { diff --git a/nemoclaw/src/shared/openshell-external-target-boundary.cts b/nemoclaw/src/shared/openshell-external-target-boundary.cts index 29334ecf020..1158ad83066 100644 --- a/nemoclaw/src/shared/openshell-external-target-boundary.cts +++ b/nemoclaw/src/shared/openshell-external-target-boundary.cts @@ -28,19 +28,24 @@ interface ExternalOpenShellTarget { } export interface OpenShellCompatibilityRange { - minVersion: string; - maxVersion: string; + readonly minVersion: string; + readonly maxVersion: string; } export interface SanitizedExternalOpenShellTargetPlan { - endpoint: string; - workspace: string; - expected_release: string; - lifecycle: "external"; - authentication_source: "file"; - ca_fingerprint: string; + readonly endpoint: string; + readonly workspace: string; + readonly expected_release: string; + readonly lifecycle: "external"; + readonly authentication_source: "file"; + readonly ca_fingerprint: string; } +type ValidatedExternalOpenShellTarget = Readonly<{ + caContents: Buffer; + plan: SanitizedExternalOpenShellTargetPlan; +}>; + type UnknownRecord = Record; const TARGET_KEYS = new Set([ @@ -307,10 +312,10 @@ function validateCaBundle(contents: Buffer): readonly Buffer[] { * Validate one explicit external target and return only its non-secret identity. * File paths remain inside this boundary, and authentication contents are not read. */ -export function buildSanitizedExternalOpenShellTargetPlan( +function validateExternalOpenShellTarget( value: unknown, compatibility: OpenShellCompatibilityRange, -): SanitizedExternalOpenShellTargetPlan { +): ValidatedExternalOpenShellTarget { const target = parseTarget(value); assertCompatibleRelease(target.expected_release, compatibility); @@ -328,11 +333,34 @@ export function buildSanitizedExternalOpenShellTargetPlan( caFingerprint.update(certificate); } return { - endpoint: target.endpoint, - workspace: target.workspace, - expected_release: target.expected_release, - lifecycle: "external", - authentication_source: "file", - ca_fingerprint: `sha256:${caFingerprint.digest("hex")}`, + caContents, + plan: Object.freeze({ + endpoint: target.endpoint, + workspace: target.workspace, + expected_release: target.expected_release, + lifecycle: "external", + authentication_source: "file", + ca_fingerprint: `sha256:${caFingerprint.digest("hex")}`, + }), }; } + +export function buildSanitizedExternalOpenShellTargetPlan( + value: unknown, + compatibility: OpenShellCompatibilityRange, +): SanitizedExternalOpenShellTargetPlan { + return validateExternalOpenShellTarget(value, compatibility).plan; +} + +/** + * Give one caller the validated CA bytes without exposing either input path. + * Authentication contents remain unread because this boundary inspects only their file metadata. + */ +export async function withExternalOpenShellTargetCa( + value: unknown, + compatibility: OpenShellCompatibilityRange, + useTarget: (plan: SanitizedExternalOpenShellTargetPlan, caContents: Buffer) => Promise, +): Promise { + const validated = validateExternalOpenShellTarget(value, compatibility); + return useTarget(validated.plan, Buffer.from(validated.caContents)); +} diff --git a/nemoclaw/src/shared/openshell-external-target-boundary.test.ts b/nemoclaw/src/shared/openshell-external-target-boundary.test.ts index dd3d3cee590..620fff3358c 100644 --- a/nemoclaw/src/shared/openshell-external-target-boundary.test.ts +++ b/nemoclaw/src/shared/openshell-external-target-boundary.test.ts @@ -20,7 +20,10 @@ vi.mock("node:fs", async (importOriginal) => ({ ...fsMocks, })); -import { buildSanitizedExternalOpenShellTargetPlan } from "./openshell-external-target-boundary.cjs"; +import { + buildSanitizedExternalOpenShellTargetPlan, + withExternalOpenShellTargetCa, +} from "./openshell-external-target-boundary.cjs"; const CA_FILE = "/var/run/openshell-target/private-ca.pem"; const AUTHENTICATION_FILE = "/var/run/openshell-target/private-authentication"; @@ -149,6 +152,20 @@ describe("external OpenShell target boundary", () => { expect(rendered).not.toContain(AUTHENTICATION_CONTENTS); expect(rendered).not.toContain("BEGIN CERTIFICATE"); expect(rendered).not.toMatch(/mtls|oidc/iu); + expect(Object.isFrozen(plan)).toBe(true); + }); + + it("passes validated CA bytes without reading authentication contents (#9872)", async () => { + const result = await withExternalOpenShellTargetCa( + externalTarget(), + COMPATIBILITY, + async (plan, caContents) => ({ plan, caContents }), + ); + + expect(result.plan.endpoint).toBe("https://openshell.example.test:8443"); + expect(result.caContents.toString("utf8")).toBe(CA_PEM); + expect(readFilePaths).toContain(CA_FILE); + expect(readFilePaths).not.toContain(AUTHENTICATION_FILE); }); it.each([ diff --git a/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts b/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts new file mode 100644 index 00000000000..21639bb20db --- /dev/null +++ b/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { SanitizedExternalOpenShellTargetPlan } from "./openshell-external-target-boundary.cjs"; +import { EXTERNAL_OPENSHELL_RELEASE } from "./openshell-observation-boundary.cjs"; +import { createOpenShellSdkGatewayHealthObserver } from "./openshell-gateway-health-sdk.js"; + +const TARGET: SanitizedExternalOpenShellTargetPlan = { + endpoint: "https://openshell.example.test:8443", + workspace: "default", + expected_release: EXTERNAL_OPENSHELL_RELEASE, + lifecycle: "external", + authentication_source: "file", + ca_fingerprint: `sha256:${"a".repeat(64)}`, +}; +const CA_BUNDLE = Buffer.from("public-ca-certificate"); +const SERVICE_STATUS = Object.freeze({ + UNSPECIFIED: 0, + HEALTHY: 1, + DEGRADED: 2, + UNHEALTHY: 3, +}); + +describe("OpenShell SDK gateway health observer", () => { + const health = vi.fn(); + const connect = vi.fn(); + const loadSdk = vi.fn(); + const timeoutSignal = vi.fn(); + + beforeEach(() => { + vi.clearAllMocks(); + health.mockResolvedValue({ status: SERVICE_STATUS.HEALTHY, version: "0.0.106" }); + connect.mockResolvedValue({ raw: { health } }); + loadSdk.mockResolvedValue({ connect, serviceStatus: SERVICE_STATUS }); + timeoutSignal.mockReturnValue(new AbortController().signal); + }); + + function request(overrides: Record = {}) { + return { + target: TARGET, + caBundle: CA_BUNDLE, + timeoutMs: 5_000, + ...overrides, + }; + } + + it("uses the official SDK client and its generated health client with one deadline (#9872)", async () => { + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request()); + + const signal = timeoutSignal.mock.results[0]?.value; + expect(timeoutSignal).toHaveBeenCalledWith(5_000); + expect(loadSdk).toHaveBeenCalledOnce(); + expect(connect).toHaveBeenCalledWith({ + gateway: TARGET.endpoint, + caCert: CA_BUNDLE, + }); + expect(health).toHaveBeenCalledWith({}, { signal }); + expect(result).toEqual({ ok: true, value: { status: "healthy", release: "0.0.106" } }); + expect(Object.isFrozen(result)).toBe(true); + expect(Object.isFrozen(result.ok ? result.value : {})).toBe(true); + }); + + it("starts the deadline before loading the SDK (#9872)", async () => { + loadSdk.mockImplementation(async () => { + expect(timeoutSignal).toHaveBeenCalledOnce(); + return { connect, serviceStatus: SERVICE_STATUS }; + }); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + await observer.observeHealth(request()); + + expect(connect).toHaveBeenCalledOnce(); + }); + + it.each([ + [SERVICE_STATUS.UNSPECIFIED, "unspecified"], + [SERVICE_STATUS.HEALTHY, "healthy"], + [SERVICE_STATUS.DEGRADED, "degraded"], + [SERVICE_STATUS.UNHEALTHY, "unhealthy"], + ])("maps SDK service status %s to %s (#9872)", async (sdkStatus, expected) => { + health.mockResolvedValue({ status: sdkStatus, version: "0.0.106" }); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request()); + + expect(result).toEqual({ ok: true, value: { status: expected, release: "0.0.106" } }); + }); + + it.each([ + ["missing response", null], + ["missing status", { version: "0.0.106" }], + ["unknown status", { status: 99, version: "0.0.106" }], + ["missing version", { status: SERVICE_STATUS.HEALTHY }], + ])("returns a fixed schema error for %s (#9872)", async (_name, response) => { + health.mockResolvedValue(response); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request()); + + expect(result).toEqual({ + ok: false, + error: { + kind: "schema", + message: "The external OpenShell gateway returned an invalid health response.", + }, + }); + }); + + it("keeps SDK load details out of a dependency failure (#9872)", async () => { + loadSdk.mockRejectedValue(new Error("private token from /var/run/private-authentication")); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request()); + + expect(result).toEqual({ + ok: false, + error: { kind: "dependency", message: "The approved OpenShell SDK package is unavailable." }, + }); + expect(connect).not.toHaveBeenCalled(); + }); + + it("keeps transport details out of a reachability failure (#9872)", async () => { + health.mockRejectedValue(new Error("private path /var/run/private-ca.pem")); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request()); + + expect(result).toEqual({ + ok: false, + error: { + kind: "transport", + message: "NemoClaw could not reach the external OpenShell target.", + }, + }); + }); + + it("bounds SDK loading with the same total deadline (#9872)", async () => { + const deadline = new AbortController(); + timeoutSignal.mockReturnValue(deadline.signal); + loadSdk.mockReturnValue(new Promise(() => undefined)); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const observation = observer.observeHealth(request()); + deadline.abort(); + + await expect(observation).resolves.toEqual({ + ok: false, + error: { + kind: "timeout", + message: "The external OpenShell gateway health check timed out.", + }, + }); + expect(connect).not.toHaveBeenCalled(); + expect(health).not.toHaveBeenCalled(); + }); + + it("passes the deadline signal to a stalled SDK health call (#9872)", async () => { + const deadline = new AbortController(); + timeoutSignal.mockReturnValue(deadline.signal); + health.mockReturnValue(new Promise(() => undefined)); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const observation = observer.observeHealth(request()); + await vi.waitFor(() => expect(health).toHaveBeenCalledOnce()); + deadline.abort(); + + await expect(observation).resolves.toEqual({ + ok: false, + error: { + kind: "timeout", + message: "The external OpenShell gateway health check timed out.", + }, + }); + expect(health).toHaveBeenCalledWith({}, { signal: deadline.signal }); + }); + + it.each([ + ["wrong release", { target: { ...TARGET, expected_release: "0.0.107" } }], + ["empty CA", { caBundle: Buffer.alloc(0) }], + ["zero timeout", { timeoutMs: 0 }], + ])("rejects %s before loading the SDK (#9872)", async (_name, overrides) => { + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request(overrides)); + + expect(result).toEqual({ + ok: false, + error: { + kind: "schema", + message: "The external OpenShell gateway health request is not valid.", + }, + }); + expect(timeoutSignal).not.toHaveBeenCalled(); + expect(loadSdk).not.toHaveBeenCalled(); + }); +}); diff --git a/nemoclaw/src/shared/openshell-gateway-health-sdk.ts b/nemoclaw/src/shared/openshell-gateway-health-sdk.ts new file mode 100644 index 00000000000..00aa6056b05 --- /dev/null +++ b/nemoclaw/src/shared/openshell-gateway-health-sdk.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as importedOpenShellObservationBoundary from "./openshell-observation-boundary.cjs"; +import type { + ObserveOpenShellGatewayHealthRequest, + OpenShellGatewayHealthObserver, + OpenShellGatewayHealthResult, + OpenShellGatewayHealthStatus, +} from "./openshell-observation-boundary.cjs"; + +const sourceOrGeneratedOpenShellObservationBoundary = + importedOpenShellObservationBoundary as typeof importedOpenShellObservationBoundary & { + default?: typeof importedOpenShellObservationBoundary; + }; +const { EXTERNAL_OPENSHELL_RELEASE } = + sourceOrGeneratedOpenShellObservationBoundary.default ?? + sourceOrGeneratedOpenShellObservationBoundary; + +type OpenShellConnectOptions = Readonly<{ gateway: string; caCert: Buffer }>; + +type OpenShellHealthClient = Readonly<{ + raw: Readonly<{ + health( + request: Record, + options: Readonly<{ signal: AbortSignal }>, + ): Promise; + }>; +}>; + +type OpenShellServiceStatus = Readonly<{ + UNSPECIFIED: number; + HEALTHY: number; + DEGRADED: number; + UNHEALTHY: number; +}>; + +type LoadedOpenShellSdk = Readonly<{ + connect: (options: OpenShellConnectOptions) => Promise; + serviceStatus: OpenShellServiceStatus; +}>; + +type OpenShellGatewayHealthSdkDependencies = Readonly<{ + loadSdk?: () => Promise; + timeoutSignal?: (timeoutMs: number) => AbortSignal; +}>; + +const OPENSHELL_SDK_PACKAGE = "@nvidia/openshell-sdk"; +const OPENSHELL_RAW_SDK_PACKAGE = "@nvidia/openshell-sdk/raw"; + +function isObjectLike(value: unknown): value is Record { + return (typeof value === "object" && value !== null) || typeof value === "function"; +} + +function parseOpenShellServiceStatus(value: unknown): OpenShellServiceStatus { + if ( + !isObjectLike(value) || + value.UNSPECIFIED !== 0 || + value.HEALTHY !== 1 || + value.DEGRADED !== 2 || + value.UNHEALTHY !== 3 + ) { + throw new TypeError("The OpenShell SDK service-status export is not compatible."); + } + return value as OpenShellServiceStatus; +} + +function parseLoadedOpenShellSdk(value: unknown): LoadedOpenShellSdk { + if (!isObjectLike(value) || typeof value.connect !== "function") { + throw new TypeError("The OpenShell SDK client export is not compatible."); + } + return Object.freeze({ + connect: value.connect as LoadedOpenShellSdk["connect"], + serviceStatus: parseOpenShellServiceStatus(value.serviceStatus), + }); +} + +async function loadOpenShellSdk(): Promise { + const [clientModule, rawModule] = await Promise.all([ + import(OPENSHELL_SDK_PACKAGE), + import(OPENSHELL_RAW_SDK_PACKAGE), + ]); + if (!isObjectLike(clientModule) || !isObjectLike(clientModule.OpenShellClient)) { + throw new TypeError("The OpenShell SDK client export is not compatible."); + } + const openShellClient = clientModule.OpenShellClient; + const connect = openShellClient.connect; + return parseLoadedOpenShellSdk({ + connect: + typeof connect === "function" + ? (options: OpenShellConnectOptions) => + Reflect.apply(connect, openShellClient, [options]) as Promise + : connect, + serviceStatus: isObjectLike(rawModule) ? rawModule.ServiceStatus : undefined, + }); +} + +async function withinDeadline(signal: AbortSignal, operation: () => Promise): Promise { + if (signal.aborted) throw new Error("health deadline expired"); + let rejectDeadline: ((error: Error) => void) | undefined; + const deadline = new Promise((_resolve, reject) => { + rejectDeadline = reject; + }); + const abort = () => rejectDeadline?.(new Error("health deadline expired")); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + try { + return await Promise.race([operation(), deadline]); + } finally { + signal.removeEventListener("abort", abort); + } +} + +function healthStatus( + status: unknown, + serviceStatus: OpenShellServiceStatus, +): OpenShellGatewayHealthStatus | null { + switch (status) { + case serviceStatus.UNSPECIFIED: + return "unspecified"; + case serviceStatus.HEALTHY: + return "healthy"; + case serviceStatus.DEGRADED: + return "degraded"; + case serviceStatus.UNHEALTHY: + return "unhealthy"; + default: + return null; + } +} + +function failure( + kind: "dependency" | "schema" | "timeout" | "transport", + message: string, +): OpenShellGatewayHealthResult { + return Object.freeze({ ok: false, error: Object.freeze({ kind, message }) }); +} + +/** Observe public gateway health through the exact approved OpenShell SDK. */ +export function createOpenShellSdkGatewayHealthObserver( + dependencies: OpenShellGatewayHealthSdkDependencies = {}, +): OpenShellGatewayHealthObserver { + return Object.freeze({ + async observeHealth(request: ObserveOpenShellGatewayHealthRequest) { + if ( + request.target.expected_release !== EXTERNAL_OPENSHELL_RELEASE || + !(request.caBundle instanceof Uint8Array) || + request.caBundle.byteLength === 0 || + !Number.isSafeInteger(request.timeoutMs) || + request.timeoutMs < 1 || + request.timeoutMs > 60_000 + ) { + return failure("schema", "The external OpenShell gateway health request is not valid."); + } + + const signal = (dependencies.timeoutSignal ?? AbortSignal.timeout)(request.timeoutMs); + let sdk: LoadedOpenShellSdk; + try { + sdk = parseLoadedOpenShellSdk( + await withinDeadline(signal, dependencies.loadSdk ?? loadOpenShellSdk), + ); + } catch { + return signal.aborted + ? failure("timeout", "The external OpenShell gateway health check timed out.") + : failure("dependency", "The approved OpenShell SDK package is unavailable."); + } + + try { + const client = await withinDeadline(signal, () => + sdk.connect({ + gateway: request.target.endpoint, + caCert: Buffer.from(request.caBundle), + }), + ); + const response = await withinDeadline(signal, () => client.raw.health({}, { signal })); + if (typeof response !== "object" || response === null || Array.isArray(response)) { + return failure( + "schema", + "The external OpenShell gateway returned an invalid health response.", + ); + } + const healthResponse = response as Record; + const status = healthStatus(healthResponse.status, sdk.serviceStatus); + if ( + status === null || + typeof healthResponse.version !== "string" || + healthResponse.version === "" + ) { + return failure( + "schema", + "The external OpenShell gateway returned an invalid health response.", + ); + } + return Object.freeze({ + ok: true, + value: Object.freeze({ status, release: healthResponse.version }), + }); + } catch { + return signal.aborted + ? failure("timeout", "The external OpenShell gateway health check timed out.") + : failure("transport", "NemoClaw could not reach the external OpenShell target."); + } + }, + }); +} + +export const sdkOpenShellGatewayHealthObserver = createOpenShellSdkGatewayHealthObserver(); diff --git a/nemoclaw/src/shared/openshell-observation-boundary.cts b/nemoclaw/src/shared/openshell-observation-boundary.cts new file mode 100644 index 00000000000..3ef1bb13bf5 --- /dev/null +++ b/nemoclaw/src/shared/openshell-observation-boundary.cts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// sourceOfTruth: Transport-neutral external OpenShell observation contract +// shared by the root CLI adapter surface and the Blueprint Runner. + +import type { SanitizedExternalOpenShellTargetPlan } from "./openshell-external-target-boundary.cjs"; + +export const EXTERNAL_OPENSHELL_RELEASE = "0.0.106" as const; + +export type OpenShellGatewayHealthStatus = "unspecified" | "healthy" | "degraded" | "unhealthy"; + +export type OpenShellGatewayHealthObservation = Readonly<{ + status: OpenShellGatewayHealthStatus; + release: string; +}>; + +export type OpenShellGatewayHealthError = Readonly<{ + kind: "dependency" | "schema" | "timeout" | "transport"; + message: string; +}>; + +export type OpenShellGatewayHealthResult = + | Readonly<{ ok: true; value: OpenShellGatewayHealthObservation }> + | Readonly<{ ok: false; error: OpenShellGatewayHealthError }>; + +export type ObserveOpenShellGatewayHealthRequest = Readonly<{ + target: SanitizedExternalOpenShellTargetPlan; + caBundle: Uint8Array; + timeoutMs: number; +}>; + +/** Transport-neutral public gateway health capability used by NemoClaw. */ +export interface OpenShellGatewayHealthObserver { + observeHealth( + request: ObserveOpenShellGatewayHealthRequest, + ): Promise; +} + +export type ExternalOpenShellGatewayStatus = Readonly<{ + openshell_target: SanitizedExternalOpenShellTargetPlan; + gateway: Readonly<{ + status: "healthy"; + release: typeof EXTERNAL_OPENSHELL_RELEASE; + }>; + compatibility: "compatible"; +}>; + +export type ExternalOpenShellGatewayResult = + | Readonly<{ ok: true; value: ExternalOpenShellGatewayStatus }> + | Readonly<{ ok: false; error: Readonly<{ message: string }> }>; + +const HEALTH_STATUSES = new Set([ + "unspecified", + "healthy", + "degraded", + "unhealthy", +]); + +function failure(message: string): ExternalOpenShellGatewayResult { + return Object.freeze({ ok: false, error: Object.freeze({ message }) }); +} + +function fixedObserverFailure(kind: OpenShellGatewayHealthError["kind"]): string { + switch (kind) { + case "dependency": + return `The approved OpenShell SDK ${EXTERNAL_OPENSHELL_RELEASE} is unavailable.`; + case "schema": + return "The external OpenShell gateway returned an invalid public health response."; + case "timeout": + case "transport": + return "NemoClaw could not reach the external OpenShell target."; + } +} + +/** Validate and sanitize one observation before it becomes Runner output. */ +export async function observeExternalOpenShellGatewayHealth( + observer: OpenShellGatewayHealthObserver, + request: ObserveOpenShellGatewayHealthRequest, +): Promise { + if ( + request.target.expected_release !== EXTERNAL_OPENSHELL_RELEASE || + !(request.caBundle instanceof Uint8Array) || + request.caBundle.byteLength === 0 || + request.caBundle.byteLength > 1024 * 1024 || + !Number.isSafeInteger(request.timeoutMs) || + request.timeoutMs < 1 || + request.timeoutMs > 60_000 + ) { + return failure("The external OpenShell gateway health request is not valid."); + } + + let observed: OpenShellGatewayHealthResult; + try { + observed = await observer.observeHealth( + Object.freeze({ + target: request.target, + caBundle: Uint8Array.from(request.caBundle), + timeoutMs: request.timeoutMs, + }), + ); + } catch { + return failure("The external OpenShell gateway health check failed."); + } + if (!observed.ok) return failure(fixedObserverFailure(observed.error.kind)); + + const { release, status } = observed.value; + if (typeof release !== "string" || !HEALTH_STATUSES.has(status)) { + return failure("The external OpenShell gateway returned an invalid public health response."); + } + if (release !== EXTERNAL_OPENSHELL_RELEASE) { + return failure( + `The external OpenShell gateway release does not match ${EXTERNAL_OPENSHELL_RELEASE}.`, + ); + } + if (status !== "healthy") { + return failure("The external OpenShell gateway is not healthy."); + } + + return Object.freeze({ + ok: true, + value: Object.freeze({ + openshell_target: request.target, + gateway: Object.freeze({ status, release }), + compatibility: "compatible", + }), + }); +} diff --git a/nemoclaw/src/shared/openshell-observation-boundary.test.ts b/nemoclaw/src/shared/openshell-observation-boundary.test.ts new file mode 100644 index 00000000000..dd280e6ef1d --- /dev/null +++ b/nemoclaw/src/shared/openshell-observation-boundary.test.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { SanitizedExternalOpenShellTargetPlan } from "./openshell-external-target-boundary.cjs"; +import { + EXTERNAL_OPENSHELL_RELEASE, + observeExternalOpenShellGatewayHealth, +} from "./openshell-observation-boundary.cjs"; + +const TARGET: SanitizedExternalOpenShellTargetPlan = Object.freeze({ + endpoint: "https://openshell.example.test:8443", + workspace: "default", + expected_release: EXTERNAL_OPENSHELL_RELEASE, + lifecycle: "external", + authentication_source: "file", + ca_fingerprint: `sha256:${"a".repeat(64)}`, +}); +const CA_BUNDLE = Buffer.from("public-ca-certificate"); + +describe("external OpenShell observation boundary", () => { + const observeHealth = vi.fn(); + const observer = { observeHealth }; + + beforeEach(() => { + vi.clearAllMocks(); + observeHealth.mockResolvedValue({ + ok: true, + value: { status: "healthy", release: EXTERNAL_OPENSHELL_RELEASE }, + }); + }); + + it("returns only the sanitized compatible health status (#9872)", async () => { + const result = await observeExternalOpenShellGatewayHealth(observer, { + target: TARGET, + caBundle: CA_BUNDLE, + timeoutMs: 5_000, + }); + + expect(result).toEqual({ + ok: true, + value: { + openshell_target: TARGET, + gateway: { status: "healthy", release: EXTERNAL_OPENSHELL_RELEASE }, + compatibility: "compatible", + }, + }); + expect(observeHealth).toHaveBeenCalledWith({ + target: TARGET, + caBundle: Uint8Array.from(CA_BUNDLE), + timeoutMs: 5_000, + }); + expect(observeHealth.mock.calls[0]?.[0].caBundle).not.toBe(CA_BUNDLE); + }); + + it.each([ + ["dependency", `The approved OpenShell SDK ${EXTERNAL_OPENSHELL_RELEASE} is unavailable.`], + ["schema", "The external OpenShell gateway returned an invalid public health response."], + ["timeout", "NemoClaw could not reach the external OpenShell target."], + ["transport", "NemoClaw could not reach the external OpenShell target."], + ])("replaces a %s observer detail with a fixed message (#9872)", async (kind, message) => { + observeHealth.mockResolvedValue({ + ok: false, + error: { kind, message: "private token from /var/run/private-authentication" }, + }); + + const result = await observeExternalOpenShellGatewayHealth(observer, { + target: TARGET, + caBundle: CA_BUNDLE, + timeoutMs: 5_000, + }); + + expect(result).toEqual({ ok: false, error: { message } }); + }); + + it("replaces a thrown observer detail with a fixed message (#9872)", async () => { + observeHealth.mockRejectedValue(new Error("private path /var/run/private-ca.pem")); + + const result = await observeExternalOpenShellGatewayHealth(observer, { + target: TARGET, + caBundle: CA_BUNDLE, + timeoutMs: 5_000, + }); + + expect(result).toEqual({ + ok: false, + error: { message: "The external OpenShell gateway health check failed." }, + }); + }); + + it.each([ + ["wrong release", { status: "healthy", release: "0.0.107" }, "release does not match"], + ["unhealthy", { status: "unhealthy", release: "0.0.106" }, "gateway is not healthy"], + ["invalid status", { status: "unknown", release: "0.0.106" }, "invalid public health"], + ])("rejects a %s observation with a fixed message (#9872)", async (_name, value, message) => { + observeHealth.mockResolvedValue({ ok: true, value }); + + const result = await observeExternalOpenShellGatewayHealth(observer, { + target: TARGET, + caBundle: CA_BUNDLE, + timeoutMs: 5_000, + }); + + expect(result).toEqual({ + ok: false, + error: { message: expect.stringContaining(message) }, + }); + }); + + it.each([ + ["another release", { target: { ...TARGET, expected_release: "0.0.107" } }], + ["empty CA", { caBundle: Buffer.alloc(0) }], + ["invalid timeout", { timeoutMs: 0 }], + ])("rejects %s before calling the observer (#9872)", async (_name, overrides) => { + const result = await observeExternalOpenShellGatewayHealth(observer, { + target: TARGET, + caBundle: CA_BUNDLE, + timeoutMs: 5_000, + ...overrides, + }); + + expect(result).toEqual({ + ok: false, + error: { message: "The external OpenShell gateway health request is not valid." }, + }); + expect(observeHealth).not.toHaveBeenCalled(); + }); +}); diff --git a/nemoclaw/tsconfig.shared.json b/nemoclaw/tsconfig.shared.json index 682ecb15493..559afab17a3 100644 --- a/nemoclaw/tsconfig.shared.json +++ b/nemoclaw/tsconfig.shared.json @@ -9,6 +9,7 @@ "src/shared/credential-filter-boundary.cts", "src/shared/openshell-gateway-endpoint-boundary.cts", "src/shared/openshell-external-target-boundary.cts", + "src/shared/openshell-observation-boundary.cts", "src/shared/openshell-policy-boundary.cts", "src/shared/private-networks-boundary.cts", "src/shared/sandbox-name.cts", diff --git a/nemoclaw/vitest.project.ts b/nemoclaw/vitest.project.ts index 75f1ed1d301..75ccdce084f 100644 --- a/nemoclaw/vitest.project.ts +++ b/nemoclaw/vitest.project.ts @@ -13,6 +13,10 @@ const canonicalOpenShellExternalTargetBoundary = path.resolve( import.meta.dirname, "src/shared/openshell-external-target-boundary.cts", ); +const canonicalOpenShellObservationBoundary = path.resolve( + import.meta.dirname, + "src/shared/openshell-observation-boundary.cts", +); const canonicalOpenShellPolicyBoundary = path.resolve( import.meta.dirname, "src/shared/openshell-policy-boundary.cts", @@ -68,6 +72,10 @@ const pluginVitestProjectOptions = { find: /^.*openshell-external-target-boundary\.cjs$/, replacement: canonicalOpenShellExternalTargetBoundary, }, + { + find: /^.*openshell-observation-boundary\.cjs$/, + replacement: canonicalOpenShellObservationBoundary, + }, { find: /^.*openshell-policy-boundary\.cjs$/, replacement: canonicalOpenShellPolicyBoundary, diff --git a/package-lock.json b/package-lock.json index 79af29f3614..6dc8be86bfe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "bin": { "nemo-deepagents": "bin/nemoclaw.js", "nemoclaw": "bin/nemoclaw.js", + "nemoclaw-blueprint-runner": "dist/lib/blueprint-runner.js", "nemohermes": "bin/nemohermes.js" }, "devDependencies": { @@ -49,6 +50,12 @@ }, "engines": { "node": ">=22.19.0" + }, + "optionalDependencies": { + "@bufbuild/protobuf": "2.12.1", + "@connectrpc/connect": "2.1.2", + "@connectrpc/connect-node": "2.1.2", + "@nvidia/openshell-sdk": "0.0.106" } }, "node_modules/@aws-crypto/crc32": { @@ -659,6 +666,13 @@ "node": ">=18" } }, + "node_modules/@bufbuild/protobuf": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.12.1.tgz", + "integrity": "sha512-BvAMfS6LrgZiryOAZ4pBYucu4wG/Ei/9o9DZ9akbREnMLbPJiom2i8b9C8IsKErQoiKqVhrerzt3kOT/RrzLHg==", + "license": "(Apache-2.0 AND BSD-3-Clause)", + "optional": true + }, "node_modules/@commitlint/cli": { "version": "20.5.0", "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-20.5.0.tgz", @@ -915,6 +929,30 @@ "node": ">=v18" } }, + "node_modules/@connectrpc/connect": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-2.1.2.tgz", + "integrity": "sha512-MXkBijtcX09R10Eb6sFeIetc6w6746eio6xtfuyVOH7oQAacT1X0GzMIQFux6Qy8cq3W/T5qX5Bei8YbFtmRGA==", + "license": "Apache-2.0", + "optional": true, + "peerDependencies": { + "@bufbuild/protobuf": "^2.7.0" + } + }, + "node_modules/@connectrpc/connect-node": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@connectrpc/connect-node/-/connect-node-2.1.2.tgz", + "integrity": "sha512-+i/aAOpsI8sIx1mbYp6d99zvxaUSF6t/jP9Ux9maAmjsZPgmIQ3JuIeYi0zJIP9zlCnBlJjkpPosshCgdRuThQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@bufbuild/protobuf": "^2.7.0", + "@connectrpc/connect": "2.1.2" + } + }, "node_modules/@conventional-changelog/git-client": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/@conventional-changelog/git-client/-/git-client-2.6.0.tgz", @@ -3677,6 +3715,21 @@ ], "license": "MIT" }, + "node_modules/@nvidia/openshell-sdk": { + "version": "0.0.106", + "resolved": "https://npm.pkg.github.com/download/@nvidia/openshell-sdk/0.0.106/dc32180ba1d658fc4ec309bdf89d2b162196928d", + "integrity": "sha512-dB4mLex23Pnw61caGMR2CMHQihy9bj7IK2elJJd718k3yevm+fOt/vG6dJg8/5us4la2BwcOdRwLvOia3tdwFw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@bufbuild/protobuf": "^2.2.3", + "@connectrpc/connect": "^2.0.0", + "@connectrpc/connect-node": "^2.0.0" + }, + "engines": { + "node": ">=20.3" + } + }, "node_modules/@oclif/core": { "version": "4.10.5", "resolved": "https://registry.npmjs.org/@oclif/core/-/core-4.10.5.tgz", diff --git a/package.json b/package.json index 31ff7e1f967..fbdd090c320 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "license": "Apache-2.0", "bin": { "nemoclaw": "./bin/nemoclaw.js", + "nemoclaw-blueprint-runner": "./dist/lib/blueprint-runner.js", "nemohermes": "./bin/nemohermes.js", "nemo-deepagents": "./bin/nemoclaw.js" }, @@ -60,7 +61,7 @@ "build:policy-boundary": "tsc -p nemoclaw/tsconfig.shared.json", "catalog:compile": "tsx src/lib/inference/serving/generate-catalog.ts", "catalog:check": "tsx src/lib/inference/serving/generate-catalog.ts --check", - "build:cli": "npm run prune:retired-cli && npm run build:policy-boundary && tsc -p tsconfig.src.json && node dist/lib/core/generate-build-identity.js && node dist/lib/inference/serving/generate-catalog.js && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", + "build:cli": "npm run prune:retired-cli && npm --prefix nemoclaw run build && tsc -p tsconfig.src.json && node --experimental-strip-types --no-warnings scripts/lib/package-blueprint-runner-runtime.mts && node dist/lib/core/generate-build-identity.js && node dist/lib/inference/serving/generate-catalog.js && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", "clean:cli": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", "prune:retired-cli": "node -e \"const fs = require('node:fs'); for (const prefix of ['dist/commands/deploy', 'dist/lib/actions/deploy']) for (const suffix of ['.js', '.js.map', '.d.ts', '.d.ts.map']) fs.rmSync(prefix + suffix, { force: true }); fs.rmSync('dist/lib/deploy', { recursive: true, force: true })\"", "typecheck:cli": "tsc -p tsconfig.cli.json", @@ -101,6 +102,12 @@ "undici": "8.10.0", "yaml": "2.8.3" }, + "optionalDependencies": { + "@bufbuild/protobuf": "2.12.1", + "@connectrpc/connect": "2.1.2", + "@connectrpc/connect-node": "2.1.2", + "@nvidia/openshell-sdk": "0.0.106" + }, "bundleDependencies": [ "p-retry" ], diff --git a/scripts/lib/package-blueprint-runner-runtime.mts b/scripts/lib/package-blueprint-runner-runtime.mts new file mode 100644 index 00000000000..f56e066f63a --- /dev/null +++ b/scripts/lib/package-blueprint-runner-runtime.mts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { cpSync, existsSync, lstatSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +const repositoryRoot = path.resolve(import.meta.dirname, "..", ".."); +const source = path.join(repositoryRoot, "nemoclaw", "dist"); +const destination = path.join(repositoryRoot, "dist", "nemoclaw"); +const runner = path.join(source, "blueprint", "runner.js"); + +if (!existsSync(runner) || !lstatSync(runner).isFile() || lstatSync(runner).isSymbolicLink()) { + throw new Error("The compiled Blueprint Runner is missing from the plugin build."); +} + +rmSync(destination, { force: true, recursive: true }); +mkdirSync(path.dirname(destination), { recursive: true }); +cpSync(source, destination, { dereference: true, recursive: true }); +writeFileSync(path.join(destination, "package.json"), '{"type":"module"}\n'); diff --git a/src/lib/adapters/openshell/sandbox-observer.ts b/src/lib/adapters/openshell/sandbox-observer.ts index 45102d10b3b..b51bfbac409 100644 --- a/src/lib/adapters/openshell/sandbox-observer.ts +++ b/src/lib/adapters/openshell/sandbox-observer.ts @@ -72,3 +72,27 @@ export function namedOpenShellGateway(gatewayName: string): OpenShellGatewayTarg export function selectedOpenShellGateway(): OpenShellGatewayTarget { return { kind: "selected" }; } + +// sourceOfTruth: nemoclaw/src/shared/openshell-observation-boundary.cts +// generatedBoundary: build:cli emits this typed capability before compiling +// the root adapter surface. +export { + EXTERNAL_OPENSHELL_RELEASE, + observeExternalOpenShellGatewayHealth, +} from "../../../../nemoclaw/dist/shared/openshell-observation-boundary.cjs"; + +export { + createOpenShellSdkGatewayHealthObserver, + sdkOpenShellGatewayHealthObserver, +} from "../../../../nemoclaw/dist/shared/openshell-gateway-health-sdk.js"; + +export type { + ExternalOpenShellGatewayResult, + ExternalOpenShellGatewayStatus, + ObserveOpenShellGatewayHealthRequest, + OpenShellGatewayHealthError, + OpenShellGatewayHealthObservation, + OpenShellGatewayHealthObserver, + OpenShellGatewayHealthResult, + OpenShellGatewayHealthStatus, +} from "../../../../nemoclaw/dist/shared/openshell-observation-boundary.cjs"; diff --git a/src/lib/blueprint-runner.ts b/src/lib/blueprint-runner.ts new file mode 100755 index 00000000000..0491cefff92 --- /dev/null +++ b/src/lib/blueprint-runner.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { redactForLog } from "./security/redact.js"; +import { + sdkOpenShellGatewayHealthObserver, + type OpenShellGatewayHealthObserver, +} from "./adapters/openshell/sandbox-observer.js"; + +const RUNNER_MODULE = "../nemoclaw/blueprint/runner.js"; +const MAX_DIAGNOSTIC_CHARACTERS = 1_024; +const UNSAFE_DIAGNOSTIC_CHARACTERS = + /[\u0000-\u001f\u007f-\u009f\u200b-\u200f\u2028-\u202e\u2060-\u206f\ufeff]+/gu; + +function reportFailure(error: unknown): void { + let detail = "Blueprint Runner failed."; + try { + detail = error instanceof Error ? error.message : String(error); + } catch { + // Keep the fixed fallback when an arbitrary thrown value cannot be rendered. + } + const redacted = redactForLog(detail); + const sanitized = + typeof redacted === "string" ? redacted.replace(UNSAFE_DIAGNOSTIC_CHARACTERS, " ").trim() : ""; + const message = (sanitized || "Blueprint Runner failed.").slice(0, MAX_DIAGNOSTIC_CHARACTERS); + process.stderr.write(`Error: ${message}\n`); + process.exitCode = 1; +} + +async function run(): Promise { + const runner = (await import(RUNNER_MODULE)) as Readonly<{ + main( + argv: string[], + options: Readonly<{ gatewayHealthObserver: OpenShellGatewayHealthObserver }>, + ): Promise; + }>; + await runner.main(process.argv.slice(2), { + gatewayHealthObserver: sdkOpenShellGatewayHealthObserver, + }); +} + +void run().catch(reportFailure); + +export {}; diff --git a/test/e2e/RETRY_INVENTORY.md b/test/e2e/RETRY_INVENTORY.md new file mode 100644 index 00000000000..7522c2b801b --- /dev/null +++ b/test/e2e/RETRY_INVENTORY.md @@ -0,0 +1,8 @@ + + + +# Live E2E retry inventory + +| Operation | Owner | Limit | Retry condition | Safety basis | Evidence | +| --- | --- | --- | --- | --- | --- | +| `external-gateway-health.tcp-readiness` | `openshell-gateway` | 10 attempts, one second apart | The newly started gateway listener rejects a TCP connection with `ECONNREFUSED` | The probe is read-only. Other errors stop without retry. The Blueprint Runner health operation runs once after the listener opens. | `external-gateway-readiness-retry.json` | diff --git a/test/e2e/live/external-gateway-health-helpers.ts b/test/e2e/live/external-gateway-health-helpers.ts index 28f74769eeb..763f41e649b 100644 --- a/test/e2e/live/external-gateway-health-helpers.ts +++ b/test/e2e/live/external-gateway-health-helpers.ts @@ -6,33 +6,18 @@ import fs from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import YAML from "yaml"; -import { ensureDockerDriverGatewayLocalTlsBundle } from "../../../dist/lib/onboard/docker-driver-gateway-local-tls"; +import { getDockerDriverGatewayLocalTlsBundle } from "../../../dist/lib/onboard/docker-driver-gateway-local-tls"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { OPENSHELL_V0106_QUALIFICATION } from "../fixtures/openshell-v0106-qualification.ts"; import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; +import { runBoundedRetry } from "../fixtures/retry-policy.ts"; export const EXTERNAL_GATEWAY_HEALTH_TIMEOUT_MS = 3 * 60_000; -const HEALTH_TIMEOUT_MS = 5_000; - -type OpenShellHealthClient = Readonly<{ - raw: Readonly<{ - health( - request: Record, - options: Readonly<{ signal: AbortSignal }>, - ): Promise; - }>; -}>; - -type OpenShellSdkModule = Readonly<{ - OpenShellClient: Readonly<{ - connect(options: Readonly<{ gateway: string; caCert: Buffer }>): Promise; - }>; -}>; - type ScenarioFixtures = Readonly<{ artifacts: ArtifactSink; cleanup: CleanupRegistry; @@ -58,11 +43,11 @@ function resolveGatewayBin(): string | null { return result.status === 0 && result.stdout.trim() ? result.stdout.trim() : null; } -function pickPort(): Promise { +function pickPort(host: string): Promise { return new Promise((resolve, reject) => { const server = net.createServer(); server.once("error", reject); - server.listen(0, "127.0.0.1", () => { + server.listen(0, host, () => { const address = server.address(); if (!address || typeof address === "string") { server.close(() => reject(new Error("failed to allocate a TCP port"))); @@ -77,6 +62,15 @@ function delay(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } +function externalHostAddress(): string { + for (const addresses of Object.values(os.networkInterfaces())) { + for (const address of addresses ?? []) { + if (address.family === "IPv4" && !address.internal) return address.address; + } + } + throw new Error("a non-loopback IPv4 address is required for external gateway health"); +} + async function stopGateway(gateway: ChildProcess): Promise { if (gateway.exitCode !== null) return; gateway.kill("SIGTERM"); @@ -87,45 +81,86 @@ async function stopGateway(gateway: ChildProcess): Promise { gateway.kill("SIGKILL"); } -async function loadSdk(): Promise { - const packageName: string = "@nvidia/openshell-sdk"; - const loaded = (await import(packageName)) as Partial; - if (!loaded.OpenShellClient || typeof loaded.OpenShellClient.connect !== "function") { - throw new Error("the reviewed OpenShell SDK client export is unavailable"); +function parseRunnerStatus(output: string): Record { + const jsonStart = output.indexOf("{"); + if (jsonStart < 0) throw new Error("Blueprint Runner status did not produce JSON"); + const parsed: unknown = JSON.parse(output.slice(jsonStart)); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new Error("Blueprint Runner status did not produce a JSON mapping"); } - return loaded as OpenShellSdkModule; + return parsed as Record; } -async function waitForPublicHealth(options: { - caCert: Buffer; - endpoint: string; - gateway: ChildProcess; -}): Promise> { - const deadline = Date.now() + 60_000; - const sdk = await loadSdk(); - const client = await sdk.OpenShellClient.connect({ - gateway: options.endpoint, - caCert: options.caCert, +type PortObservation = Readonly<{ errorCode?: string; ready: boolean }>; + +function observeGatewayPort(host: string, port: number): Promise { + return new Promise((resolve) => { + const socket = net.connect({ host, port }); + socket.setTimeout(1_000); + socket.once("connect", () => { + socket.destroy(); + resolve({ ready: true }); + }); + socket.once("error", (error: NodeJS.ErrnoException) => { + socket.destroy(); + resolve({ errorCode: error.code ?? "unknown", ready: false }); + }); + socket.once("timeout", () => { + socket.destroy(); + resolve({ errorCode: "timeout", ready: false }); + }); }); - while (Date.now() < deadline) { - if (options.gateway.exitCode !== null) { - throw new Error("OpenShell gateway exited before the public health check completed"); - } - try { - const result = await client.raw.health( - {}, - { signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) }, - ); - if (typeof result === "object" && result !== null && !Array.isArray(result)) { - const health = result as Record; - if (health.version) return { status: health.status, version: health.version }; +} + +async function waitForGatewayPort(options: { + address: string; + artifacts: ArtifactSink; + gateway: ChildProcess; + port: number; +}): Promise { + const execution = await runBoundedRetry({ + operation: "external-gateway-health.tcp-readiness", + owner: "openshell-gateway", + idempotence: "read-only", + maxAttempts: 10, + delayMs: 1_000, + onEvidence: async (evidence) => { + await options.artifacts.writeJson("external-gateway-readiness-retry.json", evidence); + }, + run: () => + options.gateway.exitCode === null + ? observeGatewayPort(options.address, options.port) + : Promise.resolve({ errorCode: "gateway-exited", ready: false }), + classify: (value, error) => { + if (error !== undefined || !value) { + return { outcome: "failed", failureClass: "deterministic" }; } - } catch { - // The gateway can refuse connections until its listener is ready. - } - await delay(250); + if (value.ready) return { outcome: "passed" }; + return { + outcome: "failed", + failureClass: value.errorCode === "ECONNREFUSED" ? "transient-external" : "deterministic", + }; + }, + }); + if (execution.outcome !== "passed") { + throw new Error("OpenShell gateway did not open its configured listener"); } - throw new Error("OpenShell gateway public health did not become available"); +} + +function runBlueprintRunnerHealth(blueprintRoot: string): Record { + const result = spawnSync( + process.execPath, + [path.join(process.cwd(), "dist", "lib", "blueprint-runner.js"), "status", "--external-target"], + { + encoding: "utf8", + env: { ...process.env, NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "pipe"], + timeout: 10_000, + }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + return parseRunnerStatus(result.stdout); } export async function runExternalGatewayHealthScenario({ @@ -147,12 +182,25 @@ export async function runExternalGatewayHealthScenario({ expect(version.status, `${version.stdout}\n${version.stderr}`).toBe(0); expect(`${version.stdout}\n${version.stderr}`).toContain(OPENSHELL_V0106_QUALIFICATION.version); - const port = await pickPort(); + const address = externalHostAddress(); + const port = await pickPort(address); const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-external-health-")); + artifacts.addRedactionValues([stateDir]); cleanup.add("remove external gateway health state", () => fs.rmSync(stateDir, { recursive: true, force: true }), ); - const tls = ensureDockerDriverGatewayLocalTlsBundle({ gatewayBin: gatewayBin!, stateDir }); + const tls = getDockerDriverGatewayLocalTlsBundle(stateDir); + const certificate = spawnSync( + gatewayBin!, + ["generate-certs", "--output-dir", tls.localTlsDir, "--server-san", address], + { + encoding: "utf8", + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "pipe"], + timeout: 15_000, + }, + ); + expect(certificate.status, `${certificate.stdout}\n${certificate.stderr}`).toBe(0); const configPath = path.join(stateDir, "gateway.toml"); fs.writeFileSync( configPath, @@ -161,7 +209,7 @@ export async function runExternalGatewayHealthScenario({ "version = 1", "", "[openshell.gateway]", - `bind_address = "127.0.0.1:${String(port)}"`, + `bind_address = "${address}:${String(port)}"`, "compute_drivers = []", "disable_tls = false", "", @@ -200,17 +248,45 @@ export async function runExternalGatewayHealthScenario({ cleanup.add("stop external gateway health gateway", () => stopGateway(gateway)); try { - progress.phase("observe public health through the reviewed SDK"); - const health = await waitForPublicHealth({ - caCert: fs.readFileSync(tls.caPath), - endpoint: `https://127.0.0.1:${String(port)}`, + const blueprintRoot = path.join(stateDir, "blueprint"); + const authenticationPath = path.join(stateDir, "authentication"); + fs.mkdirSync(blueprintRoot); + fs.writeFileSync(authenticationPath, "unused-public-health-credential\n", { mode: 0o600 }); + fs.writeFileSync( + path.join(blueprintRoot, "blueprint.yaml"), + YAML.stringify({ + version: "1.0.0", + min_openshell_version: OPENSHELL_V0106_QUALIFICATION.version, + max_openshell_version: OPENSHELL_V0106_QUALIFICATION.version, + openshell_target: { + endpoint: `https://${address}:${String(port)}`, + workspace: "default", + expected_release: OPENSHELL_V0106_QUALIFICATION.version, + lifecycle: "external", + trust: { ca_file: tls.caPath }, + authentication: { credential_file: authenticationPath }, + }, + }), + { mode: 0o600 }, + ); + + progress.phase("observe public health through the exact Blueprint Runner artifact"); + await waitForGatewayPort({ + address, + artifacts, gateway, + port, + }); + const status = runBlueprintRunnerHealth(blueprintRoot); + expect(status.compatibility).toBe("compatible"); + expect(status.gateway).toEqual({ + release: OPENSHELL_V0106_QUALIFICATION.version, + status: "healthy", }); - expect(health.version).toBe(OPENSHELL_V0106_QUALIFICATION.version); - expect(health.status).toBe(1); await artifacts.writeJson("external-gateway-health.json", { expectedRelease: OPENSHELL_V0106_QUALIFICATION.version, - reportedRelease: health.version, + reportedRelease: OPENSHELL_V0106_QUALIFICATION.version, + runner: "dist/lib/blueprint-runner.js", status: "healthy", transport: "https-explicit-ca", }); diff --git a/test/e2e/live/external-gateway-health.test.ts b/test/e2e/live/external-gateway-health.test.ts index 6196b15a9c3..8e5390018c3 100644 --- a/test/e2e/live/external-gateway-health.test.ts +++ b/test/e2e/live/external-gateway-health.test.ts @@ -8,14 +8,14 @@ import { } from "./external-gateway-health-helpers.ts"; test( - "OpenShell public health accepts the reviewed SDK over explicit HTTPS and CA (#9872)", + "Blueprint Runner observes OpenShell public health over explicit HTTPS and CA (#9872)", { timeout: EXTERNAL_GATEWAY_HEALTH_TIMEOUT_MS, meta: { e2ePhases: [ "confirm the exact OpenShell gateway and SDK prerequisites", "launch a TLS gateway without client-certificate authentication", - "observe public health through the reviewed SDK", + "observe public health through the exact Blueprint Runner artifact", ], }, }, diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index b5c6f000364..3b3c83a3d8d 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -228,6 +228,7 @@ }, { "live": "test/e2e/live/external-gateway-health.test.ts", + "liveSources": ["test/e2e/live/external-gateway-health-helpers.ts"], "fast": [ "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", "test/install/reviewed-npm-archive.test.ts" diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index e33a0b6da1f..f74be2f97b0 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -109,7 +109,8 @@ function writeCliArchive( ): void { const dist = path.join(context.payloadRoot, "dist"); const shared = path.join(context.payloadRoot, "nemoclaw", "dist", "shared"); - fs.mkdirSync(dist); + fs.mkdirSync(path.join(dist, "lib"), { recursive: true }); + fs.mkdirSync(path.join(dist, "nemoclaw", "blueprint"), { recursive: true }); fs.mkdirSync(shared, { recursive: true }); fs.writeFileSync( path.join(dist, "nemoclaw.js"), @@ -122,7 +123,12 @@ function writeCliArchive( sourceRevision: context.buildIdentitySha, })}\n`, ); + fs.writeFileSync(path.join(dist, "lib", "blueprint-runner.js"), "export {};\n"); + fs.writeFileSync(path.join(dist, "nemoclaw", "package.json"), '{"type":"module"}\n'); + fs.writeFileSync(path.join(dist, "nemoclaw", "blueprint", "runner.js"), "export {};\n"); + fs.writeFileSync(path.join(shared, "openshell-gateway-health-sdk.js"), "export {};\n"); for (const boundary of [ + "openshell-observation-boundary.cjs", "openshell-policy-boundary.cjs", "sandbox-name.cjs", "snapshot-sanitizer-boundary.cjs", @@ -645,6 +651,17 @@ describe("exact-commit CLI artifact workflow boundary", () => { path.join(fixture.workspace, "nemoclaw", "dist", "shared", "sandbox-name.cjs"), ), ).toBe(true); + expect( + fs.existsSync(path.join(fixture.workspace, "dist", "lib", "blueprint-runner.js")), + ).toBe(true); + expect( + fs.existsSync(path.join(fixture.workspace, "dist", "nemoclaw", "blueprint", "runner.js")), + ).toBe(true); + expect( + JSON.parse( + fs.readFileSync(path.join(fixture.workspace, "dist", "nemoclaw", "package.json"), "utf8"), + ), + ).toEqual({ type: "module" }); } finally { fixture.cleanup(); } @@ -741,6 +758,11 @@ describe("exact-commit CLI artifact workflow boundary", () => { path.join(fixture.workspace, "nemoclaw", "dist", "shared", "sandbox-name.cjs"), ), ).toBe(true); + expect( + JSON.parse( + fs.readFileSync(path.join(fixture.workspace, "dist", "nemoclaw", "package.json"), "utf8"), + ), + ).toEqual({ type: "module" }); expect( fs @@ -891,7 +913,13 @@ describe("exact-commit CLI artifact workflow boundary", () => { CONTENT_ADDRESSED_ARTIFACT_NAME, UNBOUND_ARTIFACT_NAME, ); - packageStep.run = packageStep.run!.replace("sandbox-name.cjs", "missing-boundary.cjs"); + packageStep.run = packageStep + .run!.replace("openshell-gateway-health-sdk.js", "missing-gateway-health-sdk.js") + .replace("openshell-observation-boundary.cjs", "missing-observation-boundary.cjs") + .replace("sandbox-name.cjs", "missing-boundary.cjs"); + packageStep.run = packageStep + .run!.replaceAll("dist/nemoclaw/package.json", "dist/nemoclaw/missing-package.json") + .replace('.type == "module"', '.type == "commonjs"'); const uploadStep = requireStep(workflow, "generate-matrix", CLI_ARTIFACT_PUBLISH_STEP); uploadStep.uses = "actions/upload-artifact@v7"; @@ -900,7 +928,11 @@ describe("exact-commit CLI artifact workflow boundary", () => { "generate-matrix must expose exact cli_artifact_provenance provenance", "CLI artifact package step must bind candidate and trusted workflow identities explicitly", `CLI artifact package step must contain ${CONTENT_ADDRESSED_ARTIFACT_NAME}`, + "CLI artifact package step must contain openshell-gateway-health-sdk.js", + "CLI artifact package step must contain openshell-observation-boundary.cjs", "CLI artifact package step must contain sandbox-name.cjs", + "CLI artifact package step must contain dist/nemoclaw/package.json", + 'CLI artifact package step must contain .type == "module"', "CLI artifact upload must use the immutable content-addressed upload contract", ]), ); diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index 0c7a2aa680f..d3cc886c9c7 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -5,15 +5,39 @@ import { describe, expect, it } from "vitest"; import { readExternalGatewayHealthWorkflow, + validateExternalGatewayHealthHelper, validateExternalGatewayHealthWorkflow, validateExternalGatewayHealthWorkflowBoundary, } from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; +import { readRepoText } from "../../helpers/e2e-workflow-contract"; describe("external gateway health workflow boundary", () => { it("accepts the checked-in trusted package and live-test contract", () => { expect(validateExternalGatewayHealthWorkflowBoundary()).toEqual([]); }); + it("rejects a helper that bypasses the exact Runner or weakens its live boundary", () => { + const source = readRepoText("test/e2e/live/external-gateway-health-helpers.ts") + .replace('"--external-target"', '"--managed-target"') + .replace("const address = externalHostAddress();", 'const address = "127.0.0.1";') + .replace("maxAttempts: 10,", "maxAttempts: 100,") + .replace( + 'runner: "dist/lib/blueprint-runner.js"', + 'runner: "test/e2e/live/external-gateway-health-helpers.ts"', + ) + .replace("artifacts.addRedactionValues([stateDir]);", "") + .concat('\nimport "@nvidia/openshell-sdk";\n'); + + expect(validateExternalGatewayHealthHelper(source)).toEqual([ + "external gateway health helper must run exact Blueprint Runner external status", + "external gateway health helper must bind the gateway certificate to a non-loopback address", + "external gateway health helper must retain the bounded readiness retry", + "external gateway health helper must record the exact Runner artifact identity", + "external gateway health helper must redact its private fixture path", + "external gateway health helper must not bypass the Runner with a direct SDK import", + ]); + }); + it("rejects package credentials or untrusted candidate execution in the package job", () => { const workflow = readExternalGatewayHealthWorkflow(); const job = workflow.jobs["package-openshell-sdk"]; diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 915f1b2c467..6fa2bb6ced9 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync, type SpawnSyncReturns } from "node:child_process"; import { createHash, X509Certificate } from "node:crypto"; -import { execFileSync, spawnSync, type SpawnSyncReturns } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -12,10 +12,13 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; const REPOSITORY_ROOT = path.join(import.meta.dirname, "..", ".."); -const PLUGIN_SOURCE = path.join(REPOSITORY_ROOT, "nemoclaw"); +const OPENSHELL_SDK_PACKAGE = "@nvidia/openshell-sdk"; const PRIVATE_AUTHENTICATION_CONTENTS = "opaque-private-authentication-material"; const PRIVATE_AMBIENT_CONTENTS = "opaque-ambient-gateway-material"; +const PRIVATE_ENTRY_SECRET = `nvapi-${"A".repeat(64)}`; const CA_PEM = rootCertificates[0]!; +const UNSAFE_DIAGNOSTIC_CHARACTERS = + /[\u0000-\u001f\u007f-\u009f\u200b-\u200f\u2028-\u202e\u2060-\u206f\ufeff]/u; function commandOutput(result: SpawnSyncReturns): string { return `${result.stdout ?? ""}${result.stderr ?? ""}`; @@ -25,6 +28,11 @@ function assertCommandSucceeded(result: SpawnSyncReturns, label: string) expect(result.status, `${label} failed:\n${commandOutput(result)}`).toBe(0); } +function expectStableSingleLineDiagnostic(stderr: string): void { + expect(stderr.endsWith("\n")).toBe(true); + expect(stderr.slice(0, -1)).not.toMatch(UNSAFE_DIAGNOSTIC_CHARACTERS); +} + function npmEnvironment(): NodeJS.ProcessEnv { return { ...process.env, @@ -34,6 +42,19 @@ function npmEnvironment(): NodeJS.ProcessEnv { }; } +function packagePath(root: string, packageName: string): string { + return path.join(root, "node_modules", ...packageName.split("/")); +} + +function installedPackageIdentity( + installRoot: string, + packageName: string, +): Readonly<{ name: unknown; version: unknown }> { + const manifestPath = path.join(packagePath(installRoot, packageName), "package.json"); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as Record; + return { name: manifest.name, version: manifest.version }; +} + function writeRuntimeProbe(probePath: string): void { fs.writeFileSync( probePath, @@ -42,20 +63,28 @@ import childProcess from "node:child_process"; import dns from "node:dns"; import fs from "node:fs"; import http from "node:http"; +import http2 from "node:http2"; import https from "node:https"; import { syncBuiltinESMExports } from "node:module"; import net from "node:net"; import path from "node:path"; import tls from "node:tls"; -import { pathToFileURL } from "node:url"; -const [runnerPath, blueprintRoot, ambientRoot, credentialPath] = process.argv.slice(2); const effects = []; +const ambientPrefix = path.resolve(process.env.NEMOCLAW_TEST_AMBIENT_ROOT) + path.sep; +const credentialPath = path.resolve(process.env.NEMOCLAW_TEST_AUTHENTICATION_FILE); +const evidencePath = path.resolve(process.env.NEMOCLAW_TEST_EVIDENCE_FILE); +const originalWriteFileSync = fs.writeFileSync; +const credentialDescriptors = new Set(); const forbid = (kind) => (..._args) => { effects.push(kind); throw new Error("forbidden packaged-runner effect"); }; +process.on("exit", () => { + originalWriteFileSync(evidencePath, JSON.stringify({ effects })); +}); + for (const name of ["exec", "execFile", "execFileSync", "execSync", "fork", "spawn", "spawnSync"]) { childProcess[name] = forbid("subprocess"); } @@ -64,31 +93,28 @@ for (const name of ["lookup", "resolve", "resolve4", "resolve6", "resolveAny"]) } http.get = forbid("network"); http.request = forbid("network"); +http2.connect = forbid("network"); https.get = forbid("network"); https.request = forbid("network"); net.Socket.prototype.connect = forbid("network"); tls.connect = forbid("network"); globalThis.fetch = forbid("network"); -const ambientPrefix = path.resolve(ambientRoot) + path.sep; -const resolvedCredentialPath = path.resolve(credentialPath); -const credentialDescriptors = new Set(); -const originalReaders = new Map(); for (const name of ["accessSync", "lstatSync", "readdirSync", "statSync"]) { - originalReaders.set(name, fs[name]); + const original = fs[name]; fs[name] = (...args) => { const candidate = args[0]; if (typeof candidate === "string" && path.resolve(candidate).startsWith(ambientPrefix)) { effects.push("ambient-gateway-read"); throw new Error("forbidden ambient gateway read"); } - return originalReaders.get(name)(...args); + return original(...args); }; } const originalOpenSync = fs.openSync; fs.openSync = (...args) => { const descriptor = originalOpenSync(...args); - if (typeof args[0] === "string" && path.resolve(args[0]) === resolvedCredentialPath) { + if (typeof args[0] === "string" && path.resolve(args[0]) === credentialPath) { credentialDescriptors.add(descriptor); } return descriptor; @@ -101,7 +127,7 @@ fs.closeSync = (descriptor) => { const originalReadFileSync = fs.readFileSync; fs.readFileSync = (...args) => { const candidate = args[0]; - if (typeof candidate === "string" && path.resolve(candidate) === resolvedCredentialPath) { + if (typeof candidate === "string" && path.resolve(candidate) === credentialPath) { effects.push("authentication-content-read"); throw new Error("forbidden authentication content read"); } @@ -138,131 +164,130 @@ for (const name of [ fs[name] = forbid("local-mutation"); } syncBuiltinESMExports(); - -process.env.HOME = ambientRoot; -process.env.XDG_CONFIG_HOME = path.join(ambientRoot, ".config"); -process.env.NEMOCLAW_BLUEPRINT_PATH = blueprintRoot; - -const runnerUrl = pathToFileURL(runnerPath).href; -const runner = await import(runnerUrl); -let planOutput = ""; -const originalWrite = process.stdout.write.bind(process.stdout); -process.stdout.write = (chunk) => { - planOutput += String(chunk); - return true; -}; - -let applyError = ""; -try { - await runner.main(["plan"]); - try { - await runner.main(["apply"]); - } catch (error) { - applyError = error instanceof Error ? error.message : String(error); - } -} finally { - process.stdout.write = originalWrite; -} - -originalWrite(JSON.stringify({ applyError, effects, planOutput, runnerUrl })); `, ); } +type ProbeEvidence = Readonly<{ + effects: string[]; +}>; + describe("packaged Blueprint Runner external target", () => { it( - "builds cleanly and denies network, subprocess, ambient gateway state, credential-content, and filesystem-mutation effects during target-only planning (#9872)", - { - timeout: 240_000, - }, + "executes the root package entry point and fails before effects when the SDK is absent (#9872)", + { timeout: 240_000 }, () => { const fixtureRoot = fs.mkdtempSync( path.join(os.tmpdir(), "nemoclaw-external-target-package-"), ); - const sourcePackage = path.join(fixtureRoot, "source-package"); const archiveRoot = path.join(fixtureRoot, "archive"); - const extractedRoot = path.join(fixtureRoot, "extracted"); + const installRoot = path.join(fixtureRoot, "install"); const runtimeRoot = path.join(fixtureRoot, "runtime"); const blueprintRoot = path.join(runtimeRoot, "blueprint"); const privateRoot = path.join(runtimeRoot, "private-inputs"); const ambientRoot = path.join(runtimeRoot, "ambient-home"); const privateCaPath = path.join(privateRoot, "private-ca.pem"); const privateAuthenticationPath = path.join(privateRoot, "private-authentication"); + const evidencePath = path.join(runtimeRoot, "probe-evidence.json"); try { - fs.mkdirSync(sourcePackage, { recursive: true }); - fs.copyFileSync( - path.join(PLUGIN_SOURCE, "package.json"), - path.join(sourcePackage, "package.json"), - ); - fs.copyFileSync( - path.join(PLUGIN_SOURCE, "package-lock.json"), - path.join(sourcePackage, "package-lock.json"), - ); - fs.copyFileSync( - path.join(PLUGIN_SOURCE, "tsconfig.json"), - path.join(sourcePackage, "tsconfig.json"), - ); - fs.copyFileSync( - path.join(PLUGIN_SOURCE, "tsconfig.shared.json"), - path.join(sourcePackage, "tsconfig.shared.json"), - ); - fs.cpSync(path.join(PLUGIN_SOURCE, "src"), path.join(sourcePackage, "src"), { - recursive: true, - }); - expect(fs.existsSync(path.join(sourcePackage, "dist"))).toBe(false); - - const installBuildDependencies = spawnSync("npm", ["ci", "--ignore-scripts", "--offline"], { - cwd: sourcePackage, - encoding: "utf8", - env: npmEnvironment(), - }); - assertCommandSucceeded(installBuildDependencies, "offline clean-build dependency install"); - - const build = spawnSync("npm", ["run", "build"], { - cwd: sourcePackage, - encoding: "utf8", - env: npmEnvironment(), - }); - assertCommandSucceeded(build, "clean package build"); - expect(fs.existsSync(path.join(sourcePackage, "dist", "blueprint", "runner.js"))).toBe( - true, - ); - fs.mkdirSync(archiveRoot, { recursive: true }); const pack = spawnSync( "npm", ["pack", "--ignore-scripts", "--silent", "--pack-destination", archiveRoot], - { cwd: sourcePackage, encoding: "utf8", env: npmEnvironment() }, + { cwd: REPOSITORY_ROOT, encoding: "utf8", env: npmEnvironment() }, ); - assertCommandSucceeded(pack, "package archive creation"); + assertCommandSucceeded(pack, "root package archive creation"); const archives = fs.readdirSync(archiveRoot).filter((entry) => entry.endsWith(".tgz")); expect(archives).toHaveLength(1); + const archive = path.join(archiveRoot, archives[0]!); - fs.mkdirSync(extractedRoot, { recursive: true }); - execFileSync("tar", ["-xzf", path.join(archiveRoot, archives[0]!), "-C", extractedRoot]); - const installedPackage = path.join(extractedRoot, "package"); - const installedRunner = path.join(installedPackage, "dist", "blueprint", "runner.js"); - expect(fs.existsSync(installedRunner)).toBe(true); - expect(fs.existsSync(path.join(installedPackage, "src"))).toBe(false); - - const pruneBuildDependencies = spawnSync( + fs.mkdirSync(installRoot, { recursive: true }); + const install = spawnSync( "npm", - ["prune", "--omit=dev", "--ignore-scripts", "--offline"], - { cwd: sourcePackage, encoding: "utf8", env: npmEnvironment() }, + ["install", "--ignore-scripts", "--offline", "--omit=dev", archive], + { cwd: installRoot, encoding: "utf8", env: npmEnvironment() }, + ); + assertCommandSucceeded(install, "ordinary installation with the reviewed Runner graph"); + + const installedPackage = packagePath(installRoot, "nemoclaw"); + const installedRunner = path.join( + installedPackage, + "nemoclaw", + "dist", + "blueprint", + "runner.js", + ); + const installedRuntimeRunner = path.join( + installedPackage, + "dist", + "nemoclaw", + "blueprint", + "runner.js", ); - assertCommandSucceeded(pruneBuildDependencies, "offline production dependency pruning"); - fs.cpSync( - path.join(sourcePackage, "node_modules"), - path.join(installedPackage, "node_modules"), - { recursive: true }, + const installedRootObserver = path.join( + installedPackage, + "dist", + "lib", + "adapters", + "openshell", + "sandbox-observer.js", ); - fs.rmSync(sourcePackage, { recursive: true, force: true }); + const installedSdkObserver = path.join( + installedPackage, + "dist", + "nemoclaw", + "shared", + "openshell-gateway-health-sdk.js", + ); + const installedBinary = path.join( + installRoot, + "node_modules", + ".bin", + "nemoclaw-blueprint-runner", + ); + expect(fs.existsSync(installedRunner)).toBe(true); + expect(fs.existsSync(installedRootObserver)).toBe(true); + expect(fs.existsSync(installedSdkObserver)).toBe(true); + expect(fs.realpathSync(installedBinary)).toBe( + fs.realpathSync(path.join(installedPackage, "dist", "lib", "blueprint-runner.js")), + ); + expect(fs.existsSync(path.join(installedPackage, "nemoclaw", "src"))).toBe(false); + expect(fs.existsSync(installedRuntimeRunner)).toBe(true); + expect( + JSON.parse( + fs.readFileSync( + path.join(installedPackage, "dist", "nemoclaw", "package.json"), + "utf8", + ), + ), + ).toEqual({ type: "module" }); + expect(installedPackageIdentity(installRoot, "@bufbuild/protobuf")).toEqual({ + name: "@bufbuild/protobuf", + version: "2.12.1", + }); + expect(installedPackageIdentity(installRoot, "@connectrpc/connect")).toEqual({ + name: "@connectrpc/connect", + version: "2.1.2", + }); + expect(installedPackageIdentity(installRoot, "@connectrpc/connect-node")).toEqual({ + name: "@connectrpc/connect-node", + version: "2.1.2", + }); + expect(installedPackageIdentity(installRoot, OPENSHELL_SDK_PACKAGE)).toEqual({ + name: OPENSHELL_SDK_PACKAGE, + version: "0.0.106", + }); + const installedSdkRoot = packagePath(installRoot, OPENSHELL_SDK_PACKAGE); + expect( + fs.existsSync(path.join(installedSdkRoot, "node_modules", "@bufbuild", "protobuf")), + ).toBe(false); expect( - fs - .realpathSync(path.join(installedPackage, "node_modules", "execa")) - .startsWith(`${fs.realpathSync(installedPackage)}${path.sep}`), - ).toBe(true); + fs.existsSync(path.join(installedSdkRoot, "node_modules", "@connectrpc", "connect")), + ).toBe(false); + expect( + fs.existsSync(path.join(installedSdkRoot, "node_modules", "@connectrpc", "connect-node")), + ).toBe(false); fs.mkdirSync(blueprintRoot, { recursive: true }); fs.mkdirSync(privateRoot, { recursive: true }); @@ -273,66 +298,152 @@ describe("packaged Blueprint Runner external target", () => { path.join(ambientRoot, ".config", "openshell", "gateway.env"), PRIVATE_AMBIENT_CONTENTS, ); - fs.writeFileSync( - path.join(blueprintRoot, "blueprint.yaml"), - YAML.stringify({ - version: "1.0.0", - min_openshell_version: "0.0.106", - max_openshell_version: "0.0.106", - openshell_target: { - endpoint: "https://openshell.example.test:8443", - workspace: "default", - expected_release: "0.0.106", - lifecycle: "external", - trust: { ca_file: privateCaPath }, - authentication: { credential_file: privateAuthenticationPath }, - }, - }), - ); + const blueprintFile = path.join(blueprintRoot, "blueprint.yaml"); + const validBlueprint = YAML.stringify({ + version: "1.0.0", + min_openshell_version: "0.0.106", + max_openshell_version: "0.0.106", + openshell_target: { + endpoint: "https://192.0.2.1:8443", + workspace: "default", + expected_release: "0.0.106", + lifecycle: "external", + trust: { ca_file: privateCaPath }, + authentication: { credential_file: privateAuthenticationPath }, + }, + }); + fs.writeFileSync(blueprintFile, validBlueprint); - const probePath = path.join(runtimeRoot, "probe.mjs"); + const probePath = path.join(installedPackage, "runtime-probe.mjs"); writeRuntimeProbe(probePath); - const probe = spawnSync( - process.execPath, - [probePath, installedRunner, blueprintRoot, ambientRoot, privateAuthenticationPath], - { - cwd: runtimeRoot, - encoding: "utf8", - env: { ...process.env, NODE_OPTIONS: "", NODE_PATH: "" }, - }, - ); const privateValues = [ + fixtureRoot, privateCaPath, privateAuthenticationPath, PRIVATE_AUTHENTICATION_CONTENTS, PRIVATE_AMBIENT_CONTENTS, + PRIVATE_ENTRY_SECRET, "BEGIN CERTIFICATE", ]; - const safeProbeDiagnostics = privateValues.reduce( - (output, value) => output.replaceAll(value, "[redacted]"), - commandOutput(probe), + const runRunner = (argv: string[]) => { + fs.rmSync(evidencePath, { force: true }); + const result = spawnSync(installedBinary, argv, { + cwd: runtimeRoot, + encoding: "utf8", + env: { + ...process.env, + HOME: ambientRoot, + NODE_OPTIONS: `--import=${probePath}`, + NEMOCLAW_BLUEPRINT_PATH: blueprintRoot, + NEMOCLAW_TEST_AMBIENT_ROOT: ambientRoot, + NEMOCLAW_TEST_AUTHENTICATION_FILE: privateAuthenticationPath, + NEMOCLAW_TEST_EVIDENCE_FILE: evidencePath, + XDG_CONFIG_HOME: path.join(ambientRoot, ".config"), + }, + }); + const safeDiagnostics = privateValues.reduce( + (output, value) => output.replaceAll(value, "[redacted]"), + commandOutput(result), + ); + expect( + fs.existsSync(evidencePath), + `packaged runner did not write probe evidence:\n${safeDiagnostics}`, + ).toBe(true); + const evidence = JSON.parse(fs.readFileSync(evidencePath, "utf8")) as ProbeEvidence; + return { evidence, result, safeDiagnostics }; + }; + + const hiddenBlueprint = `${blueprintFile}.hidden`; + fs.renameSync(blueprintFile, hiddenBlueprint); + const missingBlueprint = runRunner(["status", "--external-target"]); + fs.renameSync(hiddenBlueprint, blueprintFile); + expect(missingBlueprint.result.status, missingBlueprint.safeDiagnostics).toBe(1); + expect(missingBlueprint.result.stderr).toContain("blueprint.yaml not found"); + expect(missingBlueprint.result.stderr).not.toContain(blueprintRoot); + expectStableSingleLineDiagnostic(missingBlueprint.result.stderr); + expect(missingBlueprint.evidence).toEqual({ effects: [] }); + + fs.writeFileSync( + blueprintFile, + `openshell_target:\n trust:\n ca_file: [${privateCaPath}\u001b[31m`, ); - expect(probe.status, `packaged runner probe failed:\n${safeProbeDiagnostics}`).toBe(0); + const malformedBlueprint = runRunner(["status", "--external-target"]); + expect(malformedBlueprint.result.status, malformedBlueprint.safeDiagnostics).toBe(1); + expect(malformedBlueprint.result.stderr).toContain("blueprint.yaml contains invalid YAML"); + expect(malformedBlueprint.result.stderr).not.toContain(privateCaPath); + expectStableSingleLineDiagnostic(malformedBlueprint.result.stderr); + expect(malformedBlueprint.evidence).toEqual({ effects: [] }); - const result = JSON.parse(probe.stdout) as { - applyError: string; - effects: string[]; - planOutput: string; - runnerUrl: string; - }; - expect(result.runnerUrl.startsWith("file://" + installedPackage)).toBe(true); - expect(result.effects).toEqual([]); - expect(result.applyError).toBe( - "External OpenShell target apply is not available until typed readiness and inventory are implemented.", + fs.writeFileSync(blueprintFile, privateCaPath); + const invalidBlueprint = runRunner(["status", "--external-target"]); + expect(invalidBlueprint.result.status, invalidBlueprint.safeDiagnostics).toBe(1); + expect(invalidBlueprint.result.stderr).toContain( + "blueprint.yaml must contain a YAML mapping with valid nested component shapes", + ); + expect(invalidBlueprint.result.stderr).not.toContain(privateCaPath); + expectStableSingleLineDiagnostic(invalidBlueprint.result.stderr); + expect(invalidBlueprint.evidence).toEqual({ effects: [] }); + + fs.writeFileSync(blueprintFile, validBlueprint); + const invalidAction = runRunner([`\u001b[31m${privateCaPath}\u202e`]); + expect(invalidAction.result.status, invalidAction.safeDiagnostics).toBe(1); + expect(invalidAction.result.stderr).toContain( + "Unknown action. Use: plan, apply, status, reconcile, rollback, snapshots", ); + expect(invalidAction.result.stderr).not.toContain(privateCaPath); + expectStableSingleLineDiagnostic(invalidAction.result.stderr); + expect(invalidAction.evidence).toEqual({ effects: [] }); - const planStart = result.planOutput.indexOf("{"); + const installedRunnerSource = fs.readFileSync(installedRuntimeRunner, "utf8"); + const boundedEntryDiagnostic = (() => { + try { + const unsafeDetail = `api_key=${PRIVATE_ENTRY_SECRET}\u001b[31m\u202e\n${"x".repeat(5_000)}`; + fs.writeFileSync( + installedRuntimeRunner, + `export async function main() { throw new Error(${JSON.stringify(unsafeDetail)}); }\n`, + ); + return runRunner(["status", "--external-target"]); + } finally { + fs.writeFileSync(installedRuntimeRunner, installedRunnerSource); + } + })(); + expect(boundedEntryDiagnostic.result.status, boundedEntryDiagnostic.safeDiagnostics).toBe( + 1, + ); + expect(boundedEntryDiagnostic.result.stderr).toContain(""); + expect(boundedEntryDiagnostic.result.stderr).not.toContain(PRIVATE_ENTRY_SECRET); + expect(boundedEntryDiagnostic.result.stderr.length).toBeLessThanOrEqual(1_032); + expectStableSingleLineDiagnostic(boundedEntryDiagnostic.result.stderr); + expect(boundedEntryDiagnostic.evidence).toEqual({ effects: [] }); + + const installedSdk = packagePath(installRoot, OPENSHELL_SDK_PACKAGE); + const disabledSdk = `${installedSdk}.disabled`; + fs.renameSync(installedSdk, disabledSdk); + const absentStatus = runRunner(["status", "--external-target"]); + fs.renameSync(disabledSdk, installedSdk); + expect(absentStatus.result.status, absentStatus.safeDiagnostics).toBe(1); + expect(absentStatus.result.stderr).toContain( + "The approved OpenShell SDK 0.0.106 is unavailable.", + ); + expect(absentStatus.result.stderr).not.toContain("NemoClaw could not reach"); + expect(absentStatus.evidence).toEqual({ effects: [] }); + + const presentStatus = runRunner(["status", "--external-target"]); + expect(presentStatus.result.status, presentStatus.safeDiagnostics).toBe(1); + expect(presentStatus.result.stderr).toContain( + "NemoClaw could not reach the external OpenShell target.", + ); + expect(presentStatus.evidence).toEqual({ effects: ["network"] }); + + const plan = runRunner(["plan"]); + expect(plan.result.status, plan.safeDiagnostics).toBe(0); + expect(plan.evidence.effects).toEqual([]); + const planStart = plan.result.stdout.indexOf("{"); expect(planStart).toBeGreaterThan(-1); - const plan = JSON.parse(result.planOutput.slice(planStart)) as Record; - expect(plan).toEqual({ + expect(JSON.parse(plan.result.stdout.slice(planStart))).toEqual({ run_id: expect.stringMatching(/^nc-\d{8}-\d{6}-[0-9a-f]{8}$/u), openshell_target: { - endpoint: "https://openshell.example.test:8443", + endpoint: "https://192.0.2.1:8443", workspace: "default", expected_release: "0.0.106", lifecycle: "external", @@ -343,7 +454,34 @@ describe("packaged Blueprint Runner external target", () => { }, dry_run: false, }); - const publicOutput = `${result.planOutput}\n${result.applyError}`; + + const apply = runRunner(["apply"]); + expect(apply.result.status, apply.safeDiagnostics).toBe(1); + expect(apply.result.stderr).toContain( + "External OpenShell target apply is not available until typed readiness and inventory are implemented.", + ); + expect(apply.evidence.effects).toEqual([]); + + const publicOutput = [ + absentStatus.result.stdout, + absentStatus.result.stderr, + presentStatus.result.stdout, + presentStatus.result.stderr, + plan.result.stdout, + plan.result.stderr, + apply.result.stdout, + apply.result.stderr, + missingBlueprint.result.stdout, + missingBlueprint.result.stderr, + malformedBlueprint.result.stdout, + malformedBlueprint.result.stderr, + invalidBlueprint.result.stdout, + invalidBlueprint.result.stderr, + invalidAction.result.stdout, + invalidAction.result.stderr, + boundedEntryDiagnostic.result.stdout, + boundedEntryDiagnostic.result.stderr, + ].join("\n"); expect(privateValues.some((value) => publicOutput.includes(value))).toBe(false); } finally { fs.rmSync(fixtureRoot, { recursive: true, force: true }); diff --git a/test/package-contract/blueprint-external-target-timeout.test.ts b/test/package-contract/blueprint-external-target-timeout.test.ts new file mode 100644 index 00000000000..303eb2becb8 --- /dev/null +++ b/test/package-contract/blueprint-external-target-timeout.test.ts @@ -0,0 +1,305 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import net, { type AddressInfo, type Socket } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import tls from "node:tls"; +import { rootCertificates } from "node:tls"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +const REPOSITORY_ROOT = path.join(import.meta.dirname, "..", ".."); +const RUNNER = path.join(REPOSITORY_ROOT, "dist", "lib", "blueprint-runner.js"); +const SYSTEM_CA_PEM = rootCertificates[0]!; +const UNSAFE_DIAGNOSTIC_CHARACTERS = + /[\u0000-\u001f\u007f-\u009f\u200b-\u200f\u2028-\u202e\u2060-\u206f\ufeff]/u; + +type TlsMaterial = Readonly<{ + caPath: string; + certificatePath: string; + keyPath: string; +}>; + +function externalIpv4Address(): string { + const address = Object.values(os.networkInterfaces()) + .flat() + .find((entry) => entry?.family === "IPv4" && !entry.internal)?.address; + assert.ok(address, "the package contract requires one non-loopback IPv4 interface"); + return address; +} + +function listen(server: net.Server, host: string): Promise { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen({ host, port: 0 }, () => { + server.off("error", reject); + resolve((server.address() as AddressInfo).port); + }); + }); +} + +function close(server: net.Server): Promise { + return new Promise((resolve) => server.close(() => resolve())); +} + +function runOpenSsl(args: string[], cwd: string): void { + const result = spawnSync("openssl", args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + assert.equal(result.status, 0, "OpenSSL could not create the package TLS fixture"); +} + +function createTlsMaterial(root: string, serverSan: string): TlsMaterial { + fs.mkdirSync(root, { recursive: true }); + const caPath = path.join(root, "ca.pem"); + const caKeyPath = path.join(root, "ca-key.pem"); + const requestPath = path.join(root, "server.csr"); + const certificatePath = path.join(root, "server.pem"); + const keyPath = path.join(root, "server-key.pem"); + const extensionPath = path.join(root, "server.ext"); + + runOpenSsl( + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-sha256", + "-nodes", + "-keyout", + caKeyPath, + "-out", + caPath, + "-subj", + "/CN=NemoClaw package contract CA", + "-days", + "1", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ], + root, + ); + runOpenSsl( + [ + "req", + "-newkey", + "rsa:2048", + "-sha256", + "-nodes", + "-keyout", + keyPath, + "-out", + requestPath, + "-subj", + "/CN=NemoClaw package contract gateway", + ], + root, + ); + fs.writeFileSync( + extensionPath, + [ + "basicConstraints=critical,CA:FALSE", + "keyUsage=critical,digitalSignature,keyEncipherment", + "extendedKeyUsage=serverAuth", + `subjectAltName=IP:${serverSan}`, + "", + ].join("\n"), + ); + runOpenSsl( + [ + "x509", + "-req", + "-in", + requestPath, + "-CA", + caPath, + "-CAkey", + caKeyPath, + "-CAcreateserial", + "-out", + certificatePath, + "-days", + "1", + "-sha256", + "-extfile", + extensionPath, + ], + root, + ); + fs.chmodSync(caKeyPath, 0o600); + fs.chmodSync(keyPath, 0o600); + return { caPath, certificatePath, keyPath }; +} + +function createTlsServer(material: TlsMaterial, sockets: Set): tls.Server { + const server = tls.createServer( + { + ALPNProtocols: ["h2"], + cert: fs.readFileSync(material.certificatePath), + key: fs.readFileSync(material.keyPath), + }, + () => undefined, + ); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + server.on("tlsClientError", () => undefined); + return server; +} + +function writeExternalBlueprint( + fixtureRoot: string, + address: string, + port: number, + caContents: string, +): string { + const blueprintRoot = path.join(fixtureRoot, "blueprint"); + const caPath = path.join(fixtureRoot, "configured-ca.pem"); + const authenticationPath = path.join(fixtureRoot, "authentication"); + fs.mkdirSync(blueprintRoot, { recursive: true }); + fs.writeFileSync(caPath, caContents); + fs.writeFileSync(authenticationPath, "opaque-authentication-metadata"); + fs.writeFileSync( + path.join(blueprintRoot, "blueprint.yaml"), + YAML.stringify({ + version: "1.0.0", + min_openshell_version: "0.0.106", + max_openshell_version: "0.0.106", + openshell_target: { + endpoint: `https://${address}:${String(port)}`, + workspace: "default", + expected_release: "0.0.106", + lifecycle: "external", + trust: { ca_file: caPath }, + authentication: { credential_file: authenticationPath }, + }, + }), + ); + return blueprintRoot; +} + +function runRunner( + runtimeRoot: string, + blueprintRoot: string, +): Promise< + Readonly<{ code: number | null; durationMs: number; stderr: string; timedOut: boolean }> +> { + return new Promise((resolve) => { + const startedAt = Date.now(); + const child = spawn(process.execPath, [RUNNER, "status", "--external-target"], { + cwd: runtimeRoot, + env: { ...process.env, NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + let timedOut = false; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, 8_000); + child.once("close", (code) => { + clearTimeout(timer); + resolve({ code, durationMs: Date.now() - startedAt, stderr, timedOut }); + }); + }); +} + +function expectFixedReachabilityFailure( + result: Awaited>, + fixtureRoot: string, +): void { + expect(result.timedOut, result.stderr).toBe(false); + expect(result.code, result.stderr).toBe(1); + expect(result.durationMs).toBeLessThan(8_000); + expect(result.stderr).toContain("NemoClaw could not reach the external OpenShell target."); + expect(result.stderr).not.toContain(fixtureRoot); + expect(result.stderr).not.toContain("BEGIN CERTIFICATE"); + expect(result.stderr.endsWith("\n")).toBe(true); + expect(result.stderr.slice(0, -1)).not.toMatch(UNSAFE_DIAGNOSTIC_CHARACTERS); +} + +describe("packaged Blueprint Runner external health deadline", () => { + it( + "cancels official SDK health and exits when the TLS peer stalls (#9872)", + { timeout: 20_000 }, + async () => { + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-health-timeout-")); + const sockets = new Set(); + + try { + const address = externalIpv4Address(); + const material = createTlsMaterial(path.join(fixtureRoot, "tls"), address); + const server = createTlsServer(material, sockets); + try { + const port = await listen(server, address); + const blueprintRoot = writeExternalBlueprint( + fixtureRoot, + address, + port, + fs.readFileSync(material.caPath, "utf8"), + ); + + const result = await runRunner(fixtureRoot, blueprintRoot); + + expectFixedReachabilityFailure(result, fixtureRoot); + expect(sockets.size).toBeGreaterThan(0); + } finally { + sockets.forEach((socket) => socket.destroy()); + await close(server); + } + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } + }, + ); + + it.each([ + { name: "wrong CA", serverSan: "target", trustGeneratedCa: false }, + { name: "wrong server identity", serverSan: "192.0.2.123", trustGeneratedCa: true }, + ])("rejects a $name with fixed diagnostics (#9872)", async (testCase) => { + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-health-identity-")); + const sockets = new Set(); + + try { + const address = externalIpv4Address(); + const material = createTlsMaterial( + path.join(fixtureRoot, "tls"), + testCase.serverSan === "target" ? address : testCase.serverSan, + ); + const server = createTlsServer(material, sockets); + try { + const port = await listen(server, address); + const blueprintRoot = writeExternalBlueprint( + fixtureRoot, + address, + port, + testCase.trustGeneratedCa ? fs.readFileSync(material.caPath, "utf8") : SYSTEM_CA_PEM, + ); + + const result = await runRunner(fixtureRoot, blueprintRoot); + + expectFixedReachabilityFailure(result, fixtureRoot); + expect(result.durationMs).toBeLessThan(4_500); + } finally { + sockets.forEach((socket) => socket.destroy()); + await close(server); + } + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/test/package-contract/cli/build-upgrade.test.ts b/test/package-contract/cli/build-upgrade.test.ts index 35e0bd0fdd4..70ff1fa9d2b 100644 --- a/test/package-contract/cli/build-upgrade.test.ts +++ b/test/package-contract/cli/build-upgrade.test.ts @@ -50,9 +50,18 @@ describe("CLI source-checkout upgrade build", () => { "junction", ); symlinkSync(path.join(REPOSITORY_ROOT, "src"), path.join(fixtureRoot, "src"), "junction"); + symlinkSync( + path.join(REPOSITORY_ROOT, "scripts"), + path.join(fixtureRoot, "scripts"), + "junction", + ); const policyRoot = path.join(fixtureRoot, "nemoclaw"); mkdirSync(policyRoot); + copyFileSync( + path.join(REPOSITORY_ROOT, "nemoclaw", "package.json"), + path.join(policyRoot, "package.json"), + ); copyFileSync( path.join(REPOSITORY_ROOT, "nemoclaw", "tsconfig.json"), path.join(policyRoot, "tsconfig.json"), diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index 86cbe2158d6..492c6418ecf 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -307,13 +307,22 @@ function validateProducer(errors: string[], producer: WorkflowRecord): void { } requireFragments(errors, "CLI artifact package step", packageStep.run, [ 'git rev-parse --verify HEAD)" == "$CANDIDATE_SHA"', - "for required_file in dist/nemoclaw.js dist/build-identity.json; do", + "dist/nemoclaw.js", + "dist/build-identity.json", + "dist/lib/blueprint-runner.js", + "dist/nemoclaw/package.json", + "dist/nemoclaw/blueprint/runner.js", '[[ -f "$required_file" && ! -L "$required_file" && -s "$required_file" ]]', + "openshell-gateway-health-sdk.js", + "openshell-observation-boundary.cjs", "sandbox-name.cjs", '[[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]]', ".sourceRevision == $candidateSha", "candidate CLI build identity does not match the candidate commit SHA", + '(keys | sort) == ["type"]', + '.type == "module"', + "candidate Blueprint Runner module boundary is invalid", "--sort=name", "--mtime=@0", "nemoclaw/dist/shared", diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts index 4be42b64b22..b29cbe8eda0 100644 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ b/tools/e2e/external-gateway-health-workflow-boundary.mts @@ -11,6 +11,13 @@ import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-bou const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); +const DEFAULT_LIVE_HELPER_PATH = join( + REPO_ROOT, + "test", + "e2e", + "live", + "external-gateway-health-helpers.ts", +); const PACKAGE_JOB = "package-openshell-sdk"; const HEALTH_JOB = "external-gateway-health"; const CHECKOUT_ACTION = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; @@ -184,7 +191,7 @@ function validateHealthJob(errors: string[], job: WorkflowJob): void { E2E_JOB: "1", E2E_NON_INTERACTIVE: undefined, E2E_OBSERVABLE_OUTCOME: - "The reviewed SDK observes exact public gateway health over explicit HTTPS and CA", + "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA", E2E_TARGET_ID: HEALTH_JOB, NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106", @@ -276,8 +283,48 @@ function validateHealthJob(errors: string[], job: WorkflowJob): void { requireOrder(errors, steps, run.name ?? "", upload.name ?? ""); } +export function validateExternalGatewayHealthHelper(source: string): string[] { + const errors: string[] = []; + const required = [ + 'path.join(process.cwd(), "dist", "lib", "blueprint-runner.js")', + '"status", "--external-target"', + "NEMOCLAW_BLUEPRINT_PATH: blueprintRoot", + ]; + if (required.some((fragment) => !source.includes(fragment))) { + errors.push("external gateway health helper must run exact Blueprint Runner external status"); + } + if ( + !source.includes("const address = externalHostAddress();") || + !source.includes('bind_address = "${address}:${String(port)}"') || + !source.includes('"--server-san", address') + ) { + errors.push( + "external gateway health helper must bind the gateway certificate to a non-loopback address", + ); + } + if ( + !source.includes('operation: "external-gateway-health.tcp-readiness"') || + !source.includes("maxAttempts: 10,") + ) { + errors.push("external gateway health helper must retain the bounded readiness retry"); + } + if (!source.includes('runner: "dist/lib/blueprint-runner.js"')) { + errors.push("external gateway health helper must record the exact Runner artifact identity"); + } + if (!source.includes("artifacts.addRedactionValues([stateDir]);")) { + errors.push("external gateway health helper must redact its private fixture path"); + } + if (source.includes("@nvidia/openshell-sdk")) { + errors.push( + "external gateway health helper must not bypass the Runner with a direct SDK import", + ); + } + return errors; +} + export function validateExternalGatewayHealthWorkflow( workflow: ExternalGatewayHealthWorkflow, + liveHelperSource = readFileSync(DEFAULT_LIVE_HELPER_PATH, "utf8"), ): string[] { const errors: string[] = []; const packageJob = workflow.jobs[PACKAGE_JOB]; @@ -286,11 +333,16 @@ export function validateExternalGatewayHealthWorkflow( else validatePackageJob(errors, packageJob); if (!healthJob) errors.push(`workflow is missing ${HEALTH_JOB}`); else validateHealthJob(errors, healthJob); + errors.push(...validateExternalGatewayHealthHelper(liveHelperSource)); return errors; } export function validateExternalGatewayHealthWorkflowBoundary( workflowPath = DEFAULT_WORKFLOW_PATH, + liveHelperPath = DEFAULT_LIVE_HELPER_PATH, ): string[] { - return validateExternalGatewayHealthWorkflow(readExternalGatewayHealthWorkflow(workflowPath)); + return validateExternalGatewayHealthWorkflow( + readExternalGatewayHealthWorkflow(workflowPath), + readFileSync(liveHelperPath, "utf8"), + ); } diff --git a/vitest.config.ts b/vitest.config.ts index 288bf74e030..a839bb2fce4 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -33,6 +33,9 @@ const canonicalCredentialFilterBoundary = path.resolve( const canonicalOpenShellExternalTargetBoundary = path.resolve( "nemoclaw/src/shared/openshell-external-target-boundary.cts", ); +const canonicalOpenShellObservationBoundary = path.resolve( + "nemoclaw/src/shared/openshell-observation-boundary.cts", +); const canonicalOpenShellPolicyBoundary = path.resolve( "nemoclaw/src/shared/openshell-policy-boundary.cts", ); @@ -59,6 +62,10 @@ const canonicalSourceAliases = [ find: /^.*openshell-external-target-boundary\.cjs$/, replacement: canonicalOpenShellExternalTargetBoundary, }, + { + find: /^.*openshell-observation-boundary\.cjs$/, + replacement: canonicalOpenShellObservationBoundary, + }, { find: /^.*openshell-policy-boundary\.cjs$/, replacement: canonicalOpenShellPolicyBoundary, From a37b42d4d01ab7b515c85b8ea1e8ebf7819642fd Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 30 Aug 2026 14:31:46 -0700 Subject: [PATCH 02/28] fix(openshell): repair runner package checks Signed-off-by: Apurv Kumaria --- .../actions/ci-build-typecheck/action.yaml | 2 + .gitignore | 1 + ci/source-shape-test-budget.json | 5 ++ .../blueprint/runner-external-target.test.ts | 39 +++++++++-- .../openshell-gateway-health-sdk.test.ts | 68 +++++++++++++++++++ nemoclaw/tsconfig.runner.json | 15 ++++ nemoclaw/tsconfig.shared.json | 1 + package.json | 3 +- .../lib/package-blueprint-runner-runtime.mts | 4 +- .../pr-workflow-contract.test.ts | 7 ++ .../live/external-gateway-health-helpers.ts | 11 ++- ...l-gateway-health-workflow-boundary.test.ts | 11 +++ .../blueprint-external-target-plan.test.ts | 50 ++++++++++++-- .../blueprint-external-target-timeout.test.ts | 15 ++-- .../cli/build-upgrade.test.ts | 21 +++--- .../blueprint-runner-unsafe-diagnostic.ts | 8 +++ ...ernal-gateway-health-workflow-boundary.mts | 12 +++- 17 files changed, 242 insertions(+), 31 deletions(-) create mode 100644 nemoclaw/tsconfig.runner.json create mode 100644 test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts diff --git a/.github/actions/ci-build-typecheck/action.yaml b/.github/actions/ci-build-typecheck/action.yaml index db06972694c..53188b50dc3 100644 --- a/.github/actions/ci-build-typecheck/action.yaml +++ b/.github/actions/ci-build-typecheck/action.yaml @@ -32,6 +32,8 @@ runs: - name: Verify compiled package contracts shell: bash + env: + NPM_CONFIG_CACHE: ${{ runner.temp }}/npm run: npx vitest run --project package-contract - name: Typecheck CLI + tests diff --git a/.gitignore b/.gitignore index 45896345ee3..0280e67aa98 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ coverage/ dist/ +nemoclaw/runner-dist/ artifacts/pr-review-advisor-local/ docs/_build/ node_modules/ diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 63e0642100f..be429697204 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -191,6 +191,11 @@ "test": "provides the package token only to trusted main dependency installation", "category": "security" }, + { + "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", + "test": "reuses the trusted dependency cache for offline package contracts", + "category": "security" + }, { "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", "test": "passes only the base-packaged SDK archive to pull request dependency jobs", diff --git a/nemoclaw/src/blueprint/runner-external-target.test.ts b/nemoclaw/src/blueprint/runner-external-target.test.ts index efdd31c11ba..17af40aec23 100644 --- a/nemoclaw/src/blueprint/runner-external-target.test.ts +++ b/nemoclaw/src/blueprint/runner-external-target.test.ts @@ -205,21 +205,50 @@ describe("Blueprint Runner external OpenShell target", () => { }); it.each([ - ["run receipt", ["status", "--external-target", "--run-id", "existing"]], - ["managed profile", ["status", "--external-target", "--profile", "default"]], - ["another action", ["plan", "--external-target"]], + [ + "run receipt", + ["status", "--external-target", "--run-id", "existing"], + "--external-target and --run-id cannot be used together", + ], + [ + "managed profile", + ["status", "--external-target", "--profile", "default"], + "External target status does not accept managed-run options", + ], + [ + "another action", + ["plan", "--external-target"], + "--external-target is accepted only with status", + ], ])( "rejects external status with %s options before the health call (#9872)", - async (_name, argv) => { + async (_name, argv, message) => { seedExternalTarget(); - await expect(runMain(argv)).rejects.toThrow(); + await expect(runMain(argv)).rejects.toThrow(message); expect(observeHealth).not.toHaveBeenCalled(); expect(mockExeca).not.toHaveBeenCalled(); }, ); + it("propagates only the fixed external health failure (#9872)", async () => { + seedExternalTarget(); + observeHealth.mockResolvedValue({ + ok: false, + error: { + kind: "transport", + message: "NemoClaw could not reach the external OpenShell target.", + }, + }); + + await expect(runMain(["status", "--external-target"])).rejects.toThrow( + "NemoClaw could not reach the external OpenShell target.", + ); + + expect(mockExeca).not.toHaveBeenCalled(); + }); + it("rejects an inference endpoint override before any effect (#9872)", async () => { seedExternalTarget(); diff --git a/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts b/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts index 21639bb20db..2313ba2be7f 100644 --- a/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts +++ b/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts @@ -7,6 +7,19 @@ import type { SanitizedExternalOpenShellTargetPlan } from "./openshell-external- import { EXTERNAL_OPENSHELL_RELEASE } from "./openshell-observation-boundary.cjs"; import { createOpenShellSdkGatewayHealthObserver } from "./openshell-gateway-health-sdk.js"; +const officialSdkMocks = vi.hoisted(() => ({ + connect: vi.fn(), + health: vi.fn(), +})); + +vi.mock("@nvidia/openshell-sdk", () => ({ + OpenShellClient: { connect: officialSdkMocks.connect }, +})); + +vi.mock("@nvidia/openshell-sdk/raw", () => ({ + ServiceStatus: { UNSPECIFIED: 0, HEALTHY: 1, DEGRADED: 2, UNHEALTHY: 3 }, +})); + const TARGET: SanitizedExternalOpenShellTargetPlan = { endpoint: "https://openshell.example.test:8443", workspace: "default", @@ -35,6 +48,11 @@ describe("OpenShell SDK gateway health observer", () => { connect.mockResolvedValue({ raw: { health } }); loadSdk.mockResolvedValue({ connect, serviceStatus: SERVICE_STATUS }); timeoutSignal.mockReturnValue(new AbortController().signal); + officialSdkMocks.health.mockResolvedValue({ + status: SERVICE_STATUS.HEALTHY, + version: "0.0.106", + }); + officialSdkMocks.connect.mockResolvedValue({ raw: { health: officialSdkMocks.health } }); }); function request(overrides: Record = {}) { @@ -76,6 +94,56 @@ describe("OpenShell SDK gateway health observer", () => { expect(connect).toHaveBeenCalledOnce(); }); + it("loads the approved SDK exports through the default adapter (#9872)", async () => { + const observer = createOpenShellSdkGatewayHealthObserver({ timeoutSignal }); + + const result = await observer.observeHealth(request()); + + expect(officialSdkMocks.connect).toHaveBeenCalledWith({ + gateway: TARGET.endpoint, + caCert: CA_BUNDLE, + }); + expect(officialSdkMocks.health).toHaveBeenCalledWith( + {}, + { + signal: timeoutSignal.mock.results[0]?.value, + }, + ); + expect(result).toEqual({ ok: true, value: { status: "healthy", release: "0.0.106" } }); + }); + + it.each([ + ["missing client", null], + ["incompatible service status", { connect, serviceStatus: {} }], + ])("rejects an SDK with %s exports (#9872)", async (_name, loadedSdk) => { + loadSdk.mockResolvedValue(loadedSdk); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request()); + + expect(result).toEqual({ + ok: false, + error: { kind: "dependency", message: "The approved OpenShell SDK package is unavailable." }, + }); + expect(connect).not.toHaveBeenCalled(); + }); + + it("does not start SDK loading after an expired deadline (#9872)", async () => { + timeoutSignal.mockReturnValue(AbortSignal.abort()); + const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); + + const result = await observer.observeHealth(request()); + + expect(result).toEqual({ + ok: false, + error: { + kind: "timeout", + message: "The external OpenShell gateway health check timed out.", + }, + }); + expect(loadSdk).not.toHaveBeenCalled(); + }); + it.each([ [SERVICE_STATUS.UNSPECIFIED, "unspecified"], [SERVICE_STATUS.HEALTHY, "healthy"], diff --git a/nemoclaw/tsconfig.runner.json b/nemoclaw/tsconfig.runner.json new file mode 100644 index 00000000000..bc30a633e0e --- /dev/null +++ b/nemoclaw/tsconfig.runner.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false, + "declarationMap": false, + "outDir": "runner-dist", + "rootDir": "src", + "sourceMap": false + }, + "include": [ + "src/blueprint/runner.ts", + "src/shared/openshell-gateway-health-sdk.ts" + ], + "exclude": ["node_modules", "dist", "runner-dist", "src/**/*.test.ts"] +} diff --git a/nemoclaw/tsconfig.shared.json b/nemoclaw/tsconfig.shared.json index 559afab17a3..62212804323 100644 --- a/nemoclaw/tsconfig.shared.json +++ b/nemoclaw/tsconfig.shared.json @@ -9,6 +9,7 @@ "src/shared/credential-filter-boundary.cts", "src/shared/openshell-gateway-endpoint-boundary.cts", "src/shared/openshell-external-target-boundary.cts", + "src/shared/openshell-gateway-health-sdk.ts", "src/shared/openshell-observation-boundary.cts", "src/shared/openshell-policy-boundary.cts", "src/shared/private-networks-boundary.cts", diff --git a/package.json b/package.json index fbdd090c320..8edff3b33e4 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,8 @@ "build:policy-boundary": "tsc -p nemoclaw/tsconfig.shared.json", "catalog:compile": "tsx src/lib/inference/serving/generate-catalog.ts", "catalog:check": "tsx src/lib/inference/serving/generate-catalog.ts --check", - "build:cli": "npm run prune:retired-cli && npm --prefix nemoclaw run build && tsc -p tsconfig.src.json && node --experimental-strip-types --no-warnings scripts/lib/package-blueprint-runner-runtime.mts && node dist/lib/core/generate-build-identity.js && node dist/lib/inference/serving/generate-catalog.js && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", + "build:cli": "npm run prune:retired-cli && npm run build:policy-boundary && npm run build:runner-boundary && tsc -p tsconfig.src.json && node --experimental-strip-types --no-warnings scripts/lib/package-blueprint-runner-runtime.mts && node dist/lib/core/generate-build-identity.js && node dist/lib/inference/serving/generate-catalog.js && node dist/lib/cli/generate-oclif-metadata-manifest.js && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", + "build:runner-boundary": "node -e \"require('node:fs').rmSync('nemoclaw/runner-dist', { recursive: true, force: true })\" && tsc -p nemoclaw/tsconfig.runner.json", "clean:cli": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", "prune:retired-cli": "node -e \"const fs = require('node:fs'); for (const prefix of ['dist/commands/deploy', 'dist/lib/actions/deploy']) for (const suffix of ['.js', '.js.map', '.d.ts', '.d.ts.map']) fs.rmSync(prefix + suffix, { force: true }); fs.rmSync('dist/lib/deploy', { recursive: true, force: true })\"", "typecheck:cli": "tsc -p tsconfig.cli.json", diff --git a/scripts/lib/package-blueprint-runner-runtime.mts b/scripts/lib/package-blueprint-runner-runtime.mts index f56e066f63a..120493b8080 100644 --- a/scripts/lib/package-blueprint-runner-runtime.mts +++ b/scripts/lib/package-blueprint-runner-runtime.mts @@ -5,12 +5,12 @@ import { cpSync, existsSync, lstatSync, mkdirSync, rmSync, writeFileSync } from import path from "node:path"; const repositoryRoot = path.resolve(import.meta.dirname, "..", ".."); -const source = path.join(repositoryRoot, "nemoclaw", "dist"); +const source = path.join(repositoryRoot, "nemoclaw", "runner-dist"); const destination = path.join(repositoryRoot, "dist", "nemoclaw"); const runner = path.join(source, "blueprint", "runner.js"); if (!existsSync(runner) || !lstatSync(runner).isFile() || lstatSync(runner).isSymbolicLink()) { - throw new Error("The compiled Blueprint Runner is missing from the plugin build."); + throw new Error("The compiled blueprint runner is missing from its focused build."); } rmSync(destination, { force: true, recursive: true }); diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index dfab93ea1ee..ab47706d9b8 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -534,6 +534,13 @@ describe("pull request and main workflow contracts", () => { ); }); + // source-shape-contract: security -- Offline package checks must use only the cache populated by the trusted dependency installer + it("reuses the trusted dependency cache for offline package contracts", () => { + expect( + requiredStep(sharedActions.buildTypecheck, "Verify compiled package contracts").env, + ).toEqual({ NPM_CONFIG_CACHE: "${{ runner.temp }}/npm" }); + }); + // source-shape-contract: security -- The PR workflow must select an exact base-controlled package run before publishing its archive internally it("passes only the base-packaged SDK archive to pull request dependency jobs", () => { const packageJob = prWorkflow.jobs["openshell-sdk-package"]; diff --git a/test/e2e/live/external-gateway-health-helpers.ts b/test/e2e/live/external-gateway-health-helpers.ts index 763f41e649b..0f9aa11a317 100644 --- a/test/e2e/live/external-gateway-health-helpers.ts +++ b/test/e2e/live/external-gateway-health-helpers.ts @@ -18,6 +18,15 @@ import type { TestProgress } from "../fixtures/progress.ts"; import { runBoundedRetry } from "../fixtures/retry-policy.ts"; export const EXTERNAL_GATEWAY_HEALTH_TIMEOUT_MS = 3 * 60_000; +const BLUEPRINT_RUNNER = path.join( + import.meta.dirname, + "..", + "..", + "..", + "dist", + "lib", + "blueprint-runner.js", +); type ScenarioFixtures = Readonly<{ artifacts: ArtifactSink; cleanup: CleanupRegistry; @@ -150,7 +159,7 @@ async function waitForGatewayPort(options: { function runBlueprintRunnerHealth(blueprintRoot: string): Record { const result = spawnSync( process.execPath, - [path.join(process.cwd(), "dist", "lib", "blueprint-runner.js"), "status", "--external-target"], + [BLUEPRINT_RUNNER, "status", "--external-target"], { encoding: "utf8", env: { ...process.env, NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index d3cc886c9c7..aff88a2e08c 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -38,6 +38,17 @@ describe("external gateway health workflow boundary", () => { ]); }); + it("rejects a different module-relative Runner artifact", () => { + const source = readRepoText("test/e2e/live/external-gateway-health-helpers.ts").replace( + ' "blueprint-runner.js",', + ' "other-runner.js",', + ); + + expect(validateExternalGatewayHealthHelper(source)).toContain( + "external gateway health helper must run exact Blueprint Runner external status", + ); + }); + it("rejects package credentials or untrusted candidate execution in the package job", () => { const workflow = readExternalGatewayHealthWorkflow(); const job = workflow.jobs["package-openshell-sdk"]; diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 6fa2bb6ced9..09a35544d6e 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -13,6 +13,13 @@ import YAML from "yaml"; const REPOSITORY_ROOT = path.join(import.meta.dirname, "..", ".."); const OPENSHELL_SDK_PACKAGE = "@nvidia/openshell-sdk"; +const UNSAFE_RUNNER_FIXTURE = path.join( + REPOSITORY_ROOT, + "test", + "package-contract", + "fixtures", + "blueprint-runner-unsafe-diagnostic.ts", +); const PRIVATE_AUTHENTICATION_CONTENTS = "opaque-private-authentication-material"; const PRIVATE_AMBIENT_CONTENTS = "opaque-ambient-gateway-material"; const PRIVATE_ENTRY_SECRET = `nvapi-${"A".repeat(64)}`; @@ -33,10 +40,21 @@ function expectStableSingleLineDiagnostic(stderr: string): void { expect(stderr.slice(0, -1)).not.toMatch(UNSAFE_DIAGNOSTIC_CHARACTERS); } -function npmEnvironment(): NodeJS.ProcessEnv { +function npmEnvironment( + environment: NodeJS.ProcessEnv = process.env, + cacheExists: (candidate: string) => boolean = fs.existsSync, +): NodeJS.ProcessEnv { + const runnerCache = environment.RUNNER_TEMP + ? path.join(environment.RUNNER_TEMP, "npm") + : undefined; + const cacheDirectory = + environment.NPM_CONFIG_CACHE ?? + (runnerCache && cacheExists(path.join(runnerCache, "_cacache")) ? runnerCache : undefined) ?? + environment.npm_config_cache; return { - ...process.env, + ...environment, npm_config_audit: "false", + ...(cacheDirectory ? { npm_config_cache: cacheDirectory } : {}), npm_config_fund: "false", npm_config_update_notifier: "false", }; @@ -172,6 +190,28 @@ type ProbeEvidence = Readonly<{ effects: string[]; }>; +describe("packaged Blueprint Runner npm cache", () => { + it("uses the populated trusted runner cache before npm exec's default", () => { + const runnerTemp = path.join(os.tmpdir(), "trusted-runner"); + const environment = npmEnvironment( + { RUNNER_TEMP: runnerTemp, npm_config_cache: path.join(os.tmpdir(), "npm-default") }, + (candidate) => candidate === path.join(runnerTemp, "npm", "_cacache"), + ); + + expect(environment.npm_config_cache).toBe(path.join(runnerTemp, "npm")); + }); + + it("keeps npm exec's cache when the runner cache is not populated", () => { + const defaultCache = path.join(os.tmpdir(), "npm-default"); + const environment = npmEnvironment( + { RUNNER_TEMP: path.join(os.tmpdir(), "empty-runner"), npm_config_cache: defaultCache }, + () => false, + ); + + expect(environment.npm_config_cache).toBe(defaultCache); + }); +}); + describe("packaged Blueprint Runner external target", () => { it( "executes the root package entry point and fails before effects when the SDK is absent (#9872)", @@ -397,11 +437,7 @@ describe("packaged Blueprint Runner external target", () => { const installedRunnerSource = fs.readFileSync(installedRuntimeRunner, "utf8"); const boundedEntryDiagnostic = (() => { try { - const unsafeDetail = `api_key=${PRIVATE_ENTRY_SECRET}\u001b[31m\u202e\n${"x".repeat(5_000)}`; - fs.writeFileSync( - installedRuntimeRunner, - `export async function main() { throw new Error(${JSON.stringify(unsafeDetail)}); }\n`, - ); + fs.copyFileSync(UNSAFE_RUNNER_FIXTURE, installedRuntimeRunner); return runRunner(["status", "--external-target"]); } finally { fs.writeFileSync(installedRuntimeRunner, installedRunnerSource); diff --git a/test/package-contract/blueprint-external-target-timeout.test.ts b/test/package-contract/blueprint-external-target-timeout.test.ts index 303eb2becb8..6b466fb98cd 100644 --- a/test/package-contract/blueprint-external-target-timeout.test.ts +++ b/test/package-contract/blueprint-external-target-timeout.test.ts @@ -140,7 +140,11 @@ function createTlsMaterial(root: string, serverSan: string): TlsMaterial { return { caPath, certificatePath, keyPath }; } -function createTlsServer(material: TlsMaterial, sockets: Set): tls.Server { +function createTlsServer( + material: TlsMaterial, + sockets: Set, + observedConnections: { count: number }, +): tls.Server { const server = tls.createServer( { ALPNProtocols: ["h2"], @@ -150,6 +154,7 @@ function createTlsServer(material: TlsMaterial, sockets: Set): tls.Serve () => undefined, ); server.on("connection", (socket) => { + observedConnections.count += 1; sockets.add(socket); socket.once("close", () => sockets.delete(socket)); }); @@ -239,11 +244,12 @@ describe("packaged Blueprint Runner external health deadline", () => { async () => { const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-health-timeout-")); const sockets = new Set(); + const observedConnections = { count: 0 }; try { const address = externalIpv4Address(); const material = createTlsMaterial(path.join(fixtureRoot, "tls"), address); - const server = createTlsServer(material, sockets); + const server = createTlsServer(material, sockets, observedConnections); try { const port = await listen(server, address); const blueprintRoot = writeExternalBlueprint( @@ -256,7 +262,7 @@ describe("packaged Blueprint Runner external health deadline", () => { const result = await runRunner(fixtureRoot, blueprintRoot); expectFixedReachabilityFailure(result, fixtureRoot); - expect(sockets.size).toBeGreaterThan(0); + expect(observedConnections.count).toBeGreaterThan(0); } finally { sockets.forEach((socket) => socket.destroy()); await close(server); @@ -273,6 +279,7 @@ describe("packaged Blueprint Runner external health deadline", () => { ])("rejects a $name with fixed diagnostics (#9872)", async (testCase) => { const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-health-identity-")); const sockets = new Set(); + const observedConnections = { count: 0 }; try { const address = externalIpv4Address(); @@ -280,7 +287,7 @@ describe("packaged Blueprint Runner external health deadline", () => { path.join(fixtureRoot, "tls"), testCase.serverSan === "target" ? address : testCase.serverSan, ); - const server = createTlsServer(material, sockets); + const server = createTlsServer(material, sockets, observedConnections); try { const port = await listen(server, address); const blueprintRoot = writeExternalBlueprint( diff --git a/test/package-contract/cli/build-upgrade.test.ts b/test/package-contract/cli/build-upgrade.test.ts index 70ff1fa9d2b..4c9f3d5e387 100644 --- a/test/package-contract/cli/build-upgrade.test.ts +++ b/test/package-contract/cli/build-upgrade.test.ts @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import { + cpSync, copyFileSync, existsSync, mkdirSync, @@ -50,10 +51,11 @@ describe("CLI source-checkout upgrade build", () => { "junction", ); symlinkSync(path.join(REPOSITORY_ROOT, "src"), path.join(fixtureRoot, "src"), "junction"); - symlinkSync( - path.join(REPOSITORY_ROOT, "scripts"), - path.join(fixtureRoot, "scripts"), - "junction", + const scriptsRoot = path.join(fixtureRoot, "scripts", "lib"); + mkdirSync(scriptsRoot, { recursive: true }); + copyFileSync( + path.join(REPOSITORY_ROOT, "scripts", "lib", "package-blueprint-runner-runtime.mts"), + path.join(scriptsRoot, "package-blueprint-runner-runtime.mts"), ); const policyRoot = path.join(fixtureRoot, "nemoclaw"); @@ -70,15 +72,14 @@ describe("CLI source-checkout upgrade build", () => { path.join(REPOSITORY_ROOT, "nemoclaw", "tsconfig.shared.json"), path.join(policyRoot, "tsconfig.shared.json"), ); - symlinkSync( - path.join(REPOSITORY_ROOT, "nemoclaw", "node_modules"), - path.join(policyRoot, "node_modules"), - "junction", + copyFileSync( + path.join(REPOSITORY_ROOT, "nemoclaw", "tsconfig.runner.json"), + path.join(policyRoot, "tsconfig.runner.json"), ); - symlinkSync( + cpSync( path.join(REPOSITORY_ROOT, "nemoclaw", "src"), path.join(policyRoot, "src"), - "junction", + { recursive: true }, ); const blueprintRoot = path.join(fixtureRoot, "nemoclaw-blueprint"); diff --git a/test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts b/test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts new file mode 100644 index 00000000000..427835d6d10 --- /dev/null +++ b/test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const privateEntrySecret = `nvapi-${"A".repeat(64)}`; + +export async function main() { + throw new Error(`api_key=${privateEntrySecret}\u001b[31m\u202e\n${"x".repeat(5_000)}`); +} diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts index b29cbe8eda0..6292753b262 100644 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ b/tools/e2e/external-gateway-health-workflow-boundary.mts @@ -285,8 +285,18 @@ function validateHealthJob(errors: string[], job: WorkflowJob): void { export function validateExternalGatewayHealthHelper(source: string): string[] { const errors: string[] = []; + const exactRunnerPath = `const BLUEPRINT_RUNNER = path.join( + import.meta.dirname, + "..", + "..", + "..", + "dist", + "lib", + "blueprint-runner.js", +);`; const required = [ - 'path.join(process.cwd(), "dist", "lib", "blueprint-runner.js")', + exactRunnerPath, + "[BLUEPRINT_RUNNER, \"status\", \"--external-target\"]", '"status", "--external-target"', "NEMOCLAW_BLUEPRINT_PATH: blueprintRoot", ]; From 81c82f5f45da5aa5aa59bdca194f06da67a3acc8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 30 Aug 2026 15:37:17 -0700 Subject: [PATCH 03/28] test(openshell): lock runner package fixture Signed-off-by: Apurv Kumaria --- ci/source-architecture-budget.json | 2 +- ci/source-shape-test-budget.json | 5 ++ .../blueprint-external-target-plan.test.ts | 71 ++++++++++++++----- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 220f47d6ac4..31473c3d408 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -26,7 +26,7 @@ "src/lib/runner.ts": 85, "src/lib/security/redact.ts": 55, "src/lib/state/mcp-lifecycle-lock.ts": 21, - "src/lib/state/onboard-session.ts": 36, + "src/lib/state/onboard-session.ts": 35, "src/lib/state/registry.ts": 97, "src/lib/state/state-root.ts": 21, "src/lib/subprocess-env.ts": 24, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index be429697204..ccaf569b732 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -116,6 +116,11 @@ "test": "rejects an isolated unreviewed Deep Agents Code Node base pin", "category": "security" }, + { + "file": "test/package-contract/blueprint-external-target-plan.test.ts", + "test": "maps the package command to the reviewed Runner entry point (#9872)", + "category": "compatibility" + }, { "file": "test/agents/openclaw/openclaw-2026-6-npm-remediation.test.ts", "test": "rebuilds a guarded core archive with the patched fs-safe package bundled", diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 09a35544d6e..88208f7b9d1 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -213,6 +213,43 @@ describe("packaged Blueprint Runner npm cache", () => { }); describe("packaged Blueprint Runner external target", () => { + // source-shape-contract: compatibility -- The published package command must select the reviewed Blueprint Runner entry point + it( + "maps the package command to the reviewed Runner entry point (#9872)", + { timeout: 30_000 }, + () => { + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runner-command-")); + const archiveRoot = path.join(fixtureRoot, "archive"); + const packedRoot = path.join(fixtureRoot, "packed"); + + try { + fs.mkdirSync(archiveRoot, { recursive: true }); + fs.mkdirSync(packedRoot, { recursive: true }); + const pack = spawnSync( + "npm", + ["pack", "--ignore-scripts", "--silent", "--pack-destination", archiveRoot], + { cwd: REPOSITORY_ROOT, encoding: "utf8", env: npmEnvironment() }, + ); + assertCommandSucceeded(pack, "root package archive creation"); + const archives = fs.readdirSync(archiveRoot).filter((entry) => entry.endsWith(".tgz")); + expect(archives).toHaveLength(1); + const extract = spawnSync( + "tar", + ["-xzf", path.join(archiveRoot, archives[0]!), "--strip-components=1", "-C", packedRoot], + { encoding: "utf8", env: npmEnvironment() }, + ); + assertCommandSucceeded(extract, "root package manifest extraction"); + const manifest = JSON.parse( + fs.readFileSync(path.join(packedRoot, "package.json"), "utf8"), + ) as { bin?: Record }; + + expect(manifest.bin?.["nemoclaw-blueprint-runner"]).toBe("./dist/lib/blueprint-runner.js"); + } finally { + fs.rmSync(fixtureRoot, { force: true, recursive: true }); + } + }, + ); + it( "executes the root package entry point and fails before effects when the SDK is absent (#9872)", { timeout: 240_000 }, @@ -243,14 +280,24 @@ describe("packaged Blueprint Runner external target", () => { const archive = path.join(archiveRoot, archives[0]!); fs.mkdirSync(installRoot, { recursive: true }); - const install = spawnSync( - "npm", - ["install", "--ignore-scripts", "--offline", "--omit=dev", archive], - { cwd: installRoot, encoding: "utf8", env: npmEnvironment() }, + const extract = spawnSync( + "tar", + ["-xzf", archive, "--strip-components=1", "-C", installRoot], + { encoding: "utf8", env: npmEnvironment() }, + ); + assertCommandSucceeded(extract, "root package archive extraction"); + fs.copyFileSync( + path.join(REPOSITORY_ROOT, "package-lock.json"), + path.join(installRoot, "package-lock.json"), ); - assertCommandSucceeded(install, "ordinary installation with the reviewed Runner graph"); + const install = spawnSync("npm", ["ci", "--ignore-scripts", "--offline", "--omit=dev"], { + cwd: installRoot, + encoding: "utf8", + env: npmEnvironment(), + }); + assertCommandSucceeded(install, "locked installation of the packed Runner graph"); - const installedPackage = packagePath(installRoot, "nemoclaw"); + const installedPackage = installRoot; const installedRunner = path.join( installedPackage, "nemoclaw", @@ -280,18 +327,10 @@ describe("packaged Blueprint Runner external target", () => { "shared", "openshell-gateway-health-sdk.js", ); - const installedBinary = path.join( - installRoot, - "node_modules", - ".bin", - "nemoclaw-blueprint-runner", - ); + const installedBinary = path.join(installedPackage, "dist", "lib", "blueprint-runner.js"); expect(fs.existsSync(installedRunner)).toBe(true); expect(fs.existsSync(installedRootObserver)).toBe(true); expect(fs.existsSync(installedSdkObserver)).toBe(true); - expect(fs.realpathSync(installedBinary)).toBe( - fs.realpathSync(path.join(installedPackage, "dist", "lib", "blueprint-runner.js")), - ); expect(fs.existsSync(path.join(installedPackage, "nemoclaw", "src"))).toBe(false); expect(fs.existsSync(installedRuntimeRunner)).toBe(true); expect( @@ -367,7 +406,7 @@ describe("packaged Blueprint Runner external target", () => { ]; const runRunner = (argv: string[]) => { fs.rmSync(evidencePath, { force: true }); - const result = spawnSync(installedBinary, argv, { + const result = spawnSync(process.execPath, [installedBinary, ...argv], { cwd: runtimeRoot, encoding: "utf8", env: { From cb3c067be9863f512278b8ccf64bbaf73b292739 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 30 Aug 2026 16:22:15 -0700 Subject: [PATCH 04/28] test(openshell): stabilize runner qualification Signed-off-by: Apurv Kumaria --- ci/source-shape-test-budget.json | 2 +- test/e2e/README.md | 5 +- .../live/external-gateway-health-helpers.ts | 88 +++++++++++++------ ...l-gateway-health-workflow-boundary.test.ts | 11 +++ .../blueprint-external-target-plan.test.ts | 79 ++++++++--------- .../blueprint-external-target-timeout.test.ts | 2 +- .../ci-install-dependencies.test.ts | 1 + ...ernal-gateway-health-workflow-boundary.mts | 7 +- 8 files changed, 122 insertions(+), 73 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index ccaf569b732..6454652df23 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -118,7 +118,7 @@ }, { "file": "test/package-contract/blueprint-external-target-plan.test.ts", - "test": "maps the package command to the reviewed Runner entry point (#9872)", + "test": "executes the installed package command and fails before effects when the SDK is absent (#9872)", "category": "compatibility" }, { diff --git a/test/e2e/README.md b/test/e2e/README.md index e9731f8e034..f1435e7d958 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -280,8 +280,9 @@ npx tsx tools/e2e/credential-free-tests.mts OpenShell target work in issue #9872. The trusted workflow downloads and verifies the exact OpenShell SDK archive with package-read permission. The candidate job receives the archive but no package credential. It calls a local -OpenShell 0.0.106 gateway over HTTPS with an explicit CA. The target does not -read an authentication file or make an authenticated gateway call. +OpenShell 0.0.106 gateway over HTTPS with an explicit CA. The target validates +the authentication file metadata. It does not read the file contents or make an +authenticated gateway call. Use `jobs=external-gateway-health` for the manual pull request E2E run. The target records the expected release, reported release, public health status, diff --git a/test/e2e/live/external-gateway-health-helpers.ts b/test/e2e/live/external-gateway-health-helpers.ts index 0f9aa11a317..eb43778758c 100644 --- a/test/e2e/live/external-gateway-health-helpers.ts +++ b/test/e2e/live/external-gateway-health-helpers.ts @@ -16,8 +16,14 @@ import { OPENSHELL_V0106_QUALIFICATION } from "../fixtures/openshell-v0106-quali import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import { runBoundedRetry } from "../fixtures/retry-policy.ts"; +import { + type ShellProbe, + type ShellProbeResult, + trustedShellCommand, +} from "../fixtures/shell-probe.ts"; export const EXTERNAL_GATEWAY_HEALTH_TIMEOUT_MS = 3 * 60_000; +const UNUSED_AUTHENTICATION_SENTINEL = "unused-public-health-credential"; const BLUEPRINT_RUNNER = path.join( import.meta.dirname, "..", @@ -31,6 +37,7 @@ type ScenarioFixtures = Readonly<{ artifacts: ArtifactSink; cleanup: CleanupRegistry; progress: TestProgress; + shellProbe: ShellProbe; skip: (message?: string) => void; }>; @@ -156,19 +163,31 @@ async function waitForGatewayPort(options: { } } -function runBlueprintRunnerHealth(blueprintRoot: string): Record { - const result = spawnSync( - process.execPath, - [BLUEPRINT_RUNNER, "status", "--external-target"], +function requireProbeSuccess(result: ShellProbeResult, operation: string): void { + if (!result.timedOut && result.signal === null && result.exitCode === 0) return; + throw new Error(`${operation} failed. See the redacted E2E artifacts.`); +} + +async function runBlueprintRunnerHealth( + shellProbe: ShellProbe, + blueprintRoot: string, + privateStateRoot: string, +): Promise> { + const result = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: [BLUEPRINT_RUNNER, "status", "--external-target"], + reason: "observe public OpenShell health through the packaged Blueprint Runner", + }), { - encoding: "utf8", - env: { ...process.env, NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, - killSignal: "SIGKILL", - stdio: ["ignore", "pipe", "pipe"], - timeout: 10_000, + artifactName: "external-gateway-blueprint-runner-health", + captureLimitBytes: 64 * 1024, + env: { NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, + redactionValues: [privateStateRoot, UNUSED_AUTHENTICATION_SENTINEL], + timeoutMs: 10_000, }, ); - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + requireProbeSuccess(result, "Blueprint Runner public health observation"); return parseRunnerStatus(result.stdout); } @@ -176,20 +195,30 @@ export async function runExternalGatewayHealthScenario({ artifacts, cleanup, progress, + shellProbe, skip, }: ScenarioFixtures): Promise { const gatewayBin = resolveGatewayBin(); if (!gatewayBin) skip("openshell-gateway 0.0.106 is required"); progress.phase("confirm the exact OpenShell gateway and SDK prerequisites"); - const version = spawnSync(gatewayBin!, ["--version"], { - encoding: "utf8", - killSignal: "SIGKILL", - stdio: ["ignore", "pipe", "pipe"], - timeout: 5_000, - }); - expect(version.status, `${version.stdout}\n${version.stderr}`).toBe(0); - expect(`${version.stdout}\n${version.stderr}`).toContain(OPENSHELL_V0106_QUALIFICATION.version); + const version = await shellProbe.run( + trustedShellCommand({ + command: gatewayBin!, + args: ["--version"], + reason: "confirm the local OpenShell gateway release", + }), + { + artifactName: "external-gateway-version", + captureLimitBytes: 16 * 1024, + redactionValues: [gatewayBin!], + timeoutMs: 5_000, + }, + ); + requireProbeSuccess(version, "OpenShell gateway version check"); + if (!`${version.stdout}\n${version.stderr}`.includes(OPENSHELL_V0106_QUALIFICATION.version)) { + throw new Error("The OpenShell gateway release does not match the required release."); + } const address = externalHostAddress(); const port = await pickPort(address); @@ -199,17 +228,20 @@ export async function runExternalGatewayHealthScenario({ fs.rmSync(stateDir, { recursive: true, force: true }), ); const tls = getDockerDriverGatewayLocalTlsBundle(stateDir); - const certificate = spawnSync( - gatewayBin!, - ["generate-certs", "--output-dir", tls.localTlsDir, "--server-san", address], + const certificate = await shellProbe.run( + trustedShellCommand({ + command: gatewayBin!, + args: ["generate-certs", "--output-dir", tls.localTlsDir, "--server-san", address], + reason: "create the temporary TLS identity for the local OpenShell gateway", + }), { - encoding: "utf8", - killSignal: "SIGKILL", - stdio: ["ignore", "pipe", "pipe"], - timeout: 15_000, + artifactName: "external-gateway-generate-certs", + captureLimitBytes: 16 * 1024, + redactionValues: [gatewayBin!, stateDir], + timeoutMs: 15_000, }, ); - expect(certificate.status, `${certificate.stdout}\n${certificate.stderr}`).toBe(0); + requireProbeSuccess(certificate, "OpenShell gateway certificate generation"); const configPath = path.join(stateDir, "gateway.toml"); fs.writeFileSync( configPath, @@ -260,7 +292,7 @@ export async function runExternalGatewayHealthScenario({ const blueprintRoot = path.join(stateDir, "blueprint"); const authenticationPath = path.join(stateDir, "authentication"); fs.mkdirSync(blueprintRoot); - fs.writeFileSync(authenticationPath, "unused-public-health-credential\n", { mode: 0o600 }); + fs.writeFileSync(authenticationPath, `${UNUSED_AUTHENTICATION_SENTINEL}\n`, { mode: 0o600 }); fs.writeFileSync( path.join(blueprintRoot, "blueprint.yaml"), YAML.stringify({ @@ -286,7 +318,7 @@ export async function runExternalGatewayHealthScenario({ gateway, port, }); - const status = runBlueprintRunnerHealth(blueprintRoot); + const status = await runBlueprintRunnerHealth(shellProbe, blueprintRoot, stateDir); expect(status.compatibility).toBe("compatible"); expect(status.gateway).toEqual({ release: OPENSHELL_V0106_QUALIFICATION.version, diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index aff88a2e08c..1ea32ac5069 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -49,6 +49,17 @@ describe("external gateway health workflow boundary", () => { ); }); + it("rejects a Runner invocation that bypasses the redacted shell fixture", () => { + const source = readRepoText("test/e2e/live/external-gateway-health-helpers.ts").replace( + "const result = await shellProbe.run(", + "const result = spawnSync(", + ); + + expect(validateExternalGatewayHealthHelper(source)).toContain( + "external gateway health helper must run exact Blueprint Runner external status", + ); + }); + it("rejects package credentials or untrusted candidate execution in the package job", () => { const workflow = readExternalGatewayHealthWorkflow(); const job = workflow.jobs["package-openshell-sdk"]; diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 88208f7b9d1..927bfb32d43 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -213,51 +213,16 @@ describe("packaged Blueprint Runner npm cache", () => { }); describe("packaged Blueprint Runner external target", () => { - // source-shape-contract: compatibility -- The published package command must select the reviewed Blueprint Runner entry point + // source-shape-contract: compatibility -- The installed command must select the protected root wrapper before its runtime behavior is trusted it( - "maps the package command to the reviewed Runner entry point (#9872)", - { timeout: 30_000 }, - () => { - const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runner-command-")); - const archiveRoot = path.join(fixtureRoot, "archive"); - const packedRoot = path.join(fixtureRoot, "packed"); - - try { - fs.mkdirSync(archiveRoot, { recursive: true }); - fs.mkdirSync(packedRoot, { recursive: true }); - const pack = spawnSync( - "npm", - ["pack", "--ignore-scripts", "--silent", "--pack-destination", archiveRoot], - { cwd: REPOSITORY_ROOT, encoding: "utf8", env: npmEnvironment() }, - ); - assertCommandSucceeded(pack, "root package archive creation"); - const archives = fs.readdirSync(archiveRoot).filter((entry) => entry.endsWith(".tgz")); - expect(archives).toHaveLength(1); - const extract = spawnSync( - "tar", - ["-xzf", path.join(archiveRoot, archives[0]!), "--strip-components=1", "-C", packedRoot], - { encoding: "utf8", env: npmEnvironment() }, - ); - assertCommandSucceeded(extract, "root package manifest extraction"); - const manifest = JSON.parse( - fs.readFileSync(path.join(packedRoot, "package.json"), "utf8"), - ) as { bin?: Record }; - - expect(manifest.bin?.["nemoclaw-blueprint-runner"]).toBe("./dist/lib/blueprint-runner.js"); - } finally { - fs.rmSync(fixtureRoot, { force: true, recursive: true }); - } - }, - ); - - it( - "executes the root package entry point and fails before effects when the SDK is absent (#9872)", + "executes the installed package command and fails before effects when the SDK is absent (#9872)", { timeout: 240_000 }, () => { const fixtureRoot = fs.mkdtempSync( path.join(os.tmpdir(), "nemoclaw-external-target-package-"), ); const archiveRoot = path.join(fixtureRoot, "archive"); + const consumerRoot = path.join(fixtureRoot, "consumer"); const installRoot = path.join(fixtureRoot, "install"); const runtimeRoot = path.join(fixtureRoot, "runtime"); const blueprintRoot = path.join(runtimeRoot, "blueprint"); @@ -286,6 +251,12 @@ describe("packaged Blueprint Runner external target", () => { { encoding: "utf8", env: npmEnvironment() }, ); assertCommandSucceeded(extract, "root package archive extraction"); + const packedManifest = JSON.parse( + fs.readFileSync(path.join(installRoot, "package.json"), "utf8"), + ) as { bin?: Record }; + expect(packedManifest.bin?.["nemoclaw-blueprint-runner"]).toBe( + "./dist/lib/blueprint-runner.js", + ); fs.copyFileSync( path.join(REPOSITORY_ROOT, "package-lock.json"), path.join(installRoot, "package-lock.json"), @@ -297,6 +268,29 @@ describe("packaged Blueprint Runner external target", () => { }); assertCommandSucceeded(install, "locked installation of the packed Runner graph"); + fs.mkdirSync(consumerRoot, { recursive: true }); + fs.writeFileSync( + path.join(consumerRoot, "package.json"), + JSON.stringify({ name: "nemoclaw-blueprint-runner-consumer", private: true }), + ); + const consumerInstall = spawnSync( + "npm", + [ + "install", + "--ignore-scripts", + "--offline", + "--omit=dev", + "--no-save", + "--package-lock=false", + installRoot, + ], + { cwd: consumerRoot, encoding: "utf8", env: npmEnvironment() }, + ); + assertCommandSucceeded( + consumerInstall, + "offline consumer installation of the packed Runner", + ); + const installedPackage = installRoot; const installedRunner = path.join( installedPackage, @@ -327,7 +321,12 @@ describe("packaged Blueprint Runner external target", () => { "shared", "openshell-gateway-health-sdk.js", ); - const installedBinary = path.join(installedPackage, "dist", "lib", "blueprint-runner.js"); + const installedBinary = path.join( + consumerRoot, + "node_modules", + ".bin", + "nemoclaw-blueprint-runner", + ); expect(fs.existsSync(installedRunner)).toBe(true); expect(fs.existsSync(installedRootObserver)).toBe(true); expect(fs.existsSync(installedSdkObserver)).toBe(true); @@ -406,7 +405,7 @@ describe("packaged Blueprint Runner external target", () => { ]; const runRunner = (argv: string[]) => { fs.rmSync(evidencePath, { force: true }); - const result = spawnSync(process.execPath, [installedBinary, ...argv], { + const result = spawnSync(installedBinary, argv, { cwd: runtimeRoot, encoding: "utf8", env: { diff --git a/test/package-contract/blueprint-external-target-timeout.test.ts b/test/package-contract/blueprint-external-target-timeout.test.ts index 6b466fb98cd..ab2dde56146 100644 --- a/test/package-contract/blueprint-external-target-timeout.test.ts +++ b/test/package-contract/blueprint-external-target-timeout.test.ts @@ -229,7 +229,7 @@ function expectFixedReachabilityFailure( ): void { expect(result.timedOut, result.stderr).toBe(false); expect(result.code, result.stderr).toBe(1); - expect(result.durationMs).toBeLessThan(8_000); + expect(result.durationMs).toBeLessThan(7_000); expect(result.stderr).toContain("NemoClaw could not reach the external OpenShell target."); expect(result.stderr).not.toContain(fixtureRoot); expect(result.stderr).not.toContain("BEGIN CERTIFICATE"); diff --git a/test/repository/ci-install-dependencies.test.ts b/test/repository/ci-install-dependencies.test.ts index 359e5e8f11f..d1d3c28c455 100644 --- a/test/repository/ci-install-dependencies.test.ts +++ b/test/repository/ci-install-dependencies.test.ts @@ -60,6 +60,7 @@ describe("shared CI dependency installer", () => { NPM_CONFIG_CACHE: join(fixture.root, "npm-cache"), NPM_TRACE: fixture.trace, PATH: fixture.path, + RUNNER_TEMP: join(fixture.root, "runner-temp"), }, }); diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts index 6292753b262..ac287251415 100644 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ b/tools/e2e/external-gateway-health-workflow-boundary.mts @@ -296,9 +296,14 @@ export function validateExternalGatewayHealthHelper(source: string): string[] { );`; const required = [ exactRunnerPath, - "[BLUEPRINT_RUNNER, \"status\", \"--external-target\"]", + '[BLUEPRINT_RUNNER, "status", "--external-target"]', + `const result = await shellProbe.run( + trustedShellCommand({ + command: process.execPath,`, '"status", "--external-target"', + "captureLimitBytes: 64 * 1024", "NEMOCLAW_BLUEPRINT_PATH: blueprintRoot", + "redactionValues: [privateStateRoot, UNUSED_AUTHENTICATION_SENTINEL]", ]; if (required.some((fragment) => !source.includes(fragment))) { errors.push("external gateway health helper must run exact Blueprint Runner external status"); From 094b2db069d86d2074096acc9f18f97f8a58ec05 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 30 Aug 2026 21:54:38 -0700 Subject: [PATCH 05/28] fix(blueprint): close external health review gaps Signed-off-by: Apurv Kumaria --- ci/source-shape-test-budget.json | 15 ++++++++ .../blueprint/runner-external-target.test.ts | 1 + .../openshell-external-target-boundary.cts | 7 ++++ ...openshell-external-target-boundary.test.ts | 16 +++++++-- .../shared/openshell-observation-boundary.cts | 7 ++-- scripts/find-source-shape-tests.mts | 4 +++ ...l-gateway-health-workflow-boundary.test.ts | 17 ++++++--- .../blueprint-external-target-plan.test.ts | 35 +++++++++++++++++-- test/repository/source-shape-scanner.test.ts | 14 ++++++++ ...ernal-gateway-health-workflow-boundary.mts | 8 +++-- 10 files changed, 110 insertions(+), 14 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 6454652df23..aebff976f05 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -41,6 +41,21 @@ "test": "should return actionable unknown target error", "category": "compatibility" }, + { + "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", + "test": "rejects a helper that bypasses the exact Runner or weakens its live boundary", + "category": "security" + }, + { + "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", + "test": "rejects a different module-relative Runner artifact", + "category": "security" + }, + { + "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", + "test": "rejects a Runner invocation that bypasses the redacted shell fixture", + "category": "security" + }, { "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", "test": "keeps every candidate on standard runners when $name (#7145)", diff --git a/nemoclaw/src/blueprint/runner-external-target.test.ts b/nemoclaw/src/blueprint/runner-external-target.test.ts index 44e25bafd5d..bd01b8339cc 100644 --- a/nemoclaw/src/blueprint/runner-external-target.test.ts +++ b/nemoclaw/src/blueprint/runner-external-target.test.ts @@ -42,6 +42,7 @@ vi.mock("node:fs", async (importOriginal) => { }); vi.mock("../shared/openshell-external-target-boundary.cjs", () => ({ + EXTERNAL_OPENSHELL_RELEASE: "0.0.106", ...externalTargetBoundaryMocks, default: externalTargetBoundaryMocks, })); diff --git a/nemoclaw/src/shared/openshell-external-target-boundary.cts b/nemoclaw/src/shared/openshell-external-target-boundary.cts index 1158ad83066..59419dfca4a 100644 --- a/nemoclaw/src/shared/openshell-external-target-boundary.cts +++ b/nemoclaw/src/shared/openshell-external-target-boundary.cts @@ -32,6 +32,8 @@ export interface OpenShellCompatibilityRange { readonly maxVersion: string; } +export const EXTERNAL_OPENSHELL_RELEASE = "0.0.106" as const; + export interface SanitizedExternalOpenShellTargetPlan { readonly endpoint: string; readonly workspace: string; @@ -208,6 +210,11 @@ function assertCompatibleRelease( if (compareSemver(compatibility.minVersion, compatibility.maxVersion) > 0) { throw new Error("OpenShell compatibility range minimum must not exceed its maximum"); } + if (expectedRelease !== EXTERNAL_OPENSHELL_RELEASE) { + throw new Error( + `external OpenShell target expected_release must be ${EXTERNAL_OPENSHELL_RELEASE}`, + ); + } if ( compareSemver(expectedRelease, compatibility.minVersion) < 0 || compareSemver(expectedRelease, compatibility.maxVersion) > 0 diff --git a/nemoclaw/src/shared/openshell-external-target-boundary.test.ts b/nemoclaw/src/shared/openshell-external-target-boundary.test.ts index 620fff3358c..0c1d63f4240 100644 --- a/nemoclaw/src/shared/openshell-external-target-boundary.test.ts +++ b/nemoclaw/src/shared/openshell-external-target-boundary.test.ts @@ -260,10 +260,20 @@ describe("external OpenShell target boundary", () => { }); it("rejects an incompatible expected release before reading files (#9872)", () => { - const target = { ...externalTarget(), expected_release: "0.0.107" }; + const compatibility = { minVersion: "0.0.105", maxVersion: "0.0.105" }; - expect(() => buildSanitizedExternalOpenShellTargetPlan(target, COMPATIBILITY)).toThrow( - /outside the compatible range/, + expect(() => + buildSanitizedExternalOpenShellTargetPlan(externalTarget(), compatibility), + ).toThrow(/outside the compatible range/); + expect(fsMocks.openSync).not.toHaveBeenCalled(); + }); + + it("rejects a range-compatible release that public health does not support (#9872)", () => { + const target = { ...externalTarget(), expected_release: "0.0.105" }; + const compatibility = { minVersion: "0.0.105", maxVersion: "0.0.106" }; + + expect(() => buildSanitizedExternalOpenShellTargetPlan(target, compatibility)).toThrow( + "external OpenShell target expected_release must be 0.0.106", ); expect(fsMocks.openSync).not.toHaveBeenCalled(); }); diff --git a/nemoclaw/src/shared/openshell-observation-boundary.cts b/nemoclaw/src/shared/openshell-observation-boundary.cts index 3ef1bb13bf5..20940672575 100644 --- a/nemoclaw/src/shared/openshell-observation-boundary.cts +++ b/nemoclaw/src/shared/openshell-observation-boundary.cts @@ -4,9 +4,12 @@ // sourceOfTruth: Transport-neutral external OpenShell observation contract // shared by the root CLI adapter surface and the Blueprint Runner. -import type { SanitizedExternalOpenShellTargetPlan } from "./openshell-external-target-boundary.cjs"; +import { + EXTERNAL_OPENSHELL_RELEASE, + type SanitizedExternalOpenShellTargetPlan, +} from "./openshell-external-target-boundary.cjs"; -export const EXTERNAL_OPENSHELL_RELEASE = "0.0.106" as const; +export { EXTERNAL_OPENSHELL_RELEASE }; export type OpenShellGatewayHealthStatus = "unspecified" | "healthy" | "degraded" | "unhealthy"; diff --git a/scripts/find-source-shape-tests.mts b/scripts/find-source-shape-tests.mts index 9fdf42c8353..b90372dc34e 100755 --- a/scripts/find-source-shape-tests.mts +++ b/scripts/find-source-shape-tests.mts @@ -588,6 +588,9 @@ function collectDeclarativeImports(imports: readonly ImportBinding[]): Map> = { + "tools/e2e/external-gateway-health-workflow-boundary": [ + "readExternalGatewayHealthHelper", + ], "test/helpers/e2e-workflow-contract": ["readWorkflow", "readYaml"], "test/e2e/registry/registry": ["getTarget", "listTargets", "requireTargets"], "test/e2e/registry/expected-states": [ @@ -1823,6 +1826,7 @@ function scanSourceTextReport(fileName: string, relPath: string, text: string): const productionConsumerNames = collectProductionConsumerNames(scopedImports); const declarativeImports = collectDeclarativeImports(scopedImports); const rawConfigAccessors = collectRawConfigAccessors(scopedImports); + for (const name of rawConfigAccessors.keys()) productionConsumerNames.delete(name); const variables = [ ...scopedVariableDecls(sourceFile, allVariables, node, body), ...collectSetupHookAssignments(sourceFile, node), diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index 1ea32ac5069..1335e1c9b1a 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -4,20 +4,23 @@ import { describe, expect, it } from "vitest"; import { + readExternalGatewayHealthHelper, readExternalGatewayHealthWorkflow, validateExternalGatewayHealthHelper, validateExternalGatewayHealthWorkflow, validateExternalGatewayHealthWorkflowBoundary, } from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; -import { readRepoText } from "../../helpers/e2e-workflow-contract"; describe("external gateway health workflow boundary", () => { it("accepts the checked-in trusted package and live-test contract", () => { expect(validateExternalGatewayHealthWorkflowBoundary()).toEqual([]); }); + // source-shape-contract: security -- The live helper must keep SDK access behind the packaged Runner, bounded retry, and redacted evidence. it("rejects a helper that bypasses the exact Runner or weakens its live boundary", () => { - const source = readRepoText("test/e2e/live/external-gateway-health-helpers.ts") + const helperSource = readExternalGatewayHealthHelper(); + expect(helperSource).toContain("const BLUEPRINT_RUNNER = path.join("); + const source = helperSource .replace('"--external-target"', '"--managed-target"') .replace("const address = externalHostAddress();", 'const address = "127.0.0.1";') .replace("maxAttempts: 10,", "maxAttempts: 100,") @@ -38,8 +41,11 @@ describe("external gateway health workflow boundary", () => { ]); }); + // source-shape-contract: security -- The live helper must execute the candidate Runner module before its health evidence is trusted. it("rejects a different module-relative Runner artifact", () => { - const source = readRepoText("test/e2e/live/external-gateway-health-helpers.ts").replace( + const helperSource = readExternalGatewayHealthHelper(); + expect(helperSource).toContain(' "blueprint-runner.js",'); + const source = helperSource.replace( ' "blueprint-runner.js",', ' "other-runner.js",', ); @@ -49,8 +55,11 @@ describe("external gateway health workflow boundary", () => { ); }); + // source-shape-contract: security -- The live helper must keep Runner output behind the bounded redacted shell boundary. it("rejects a Runner invocation that bypasses the redacted shell fixture", () => { - const source = readRepoText("test/e2e/live/external-gateway-health-helpers.ts").replace( + const helperSource = readExternalGatewayHealthHelper(); + expect(helperSource).toContain("const result = await shellProbe.run("); + const source = helperSource.replace( "const result = await shellProbe.run(", "const result = spawnSync(", ); diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 927bfb32d43..2f063264154 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -284,7 +284,11 @@ describe("packaged Blueprint Runner external target", () => { "--package-lock=false", installRoot, ], - { cwd: consumerRoot, encoding: "utf8", env: npmEnvironment() }, + { + cwd: consumerRoot, + encoding: "utf8", + env: { ...npmEnvironment(), NEMOCLAW_INSTALLING: "1" }, + }, ); assertCommandSucceeded( consumerInstall, @@ -377,7 +381,7 @@ describe("packaged Blueprint Runner external target", () => { PRIVATE_AMBIENT_CONTENTS, ); const blueprintFile = path.join(blueprintRoot, "blueprint.yaml"); - const validBlueprint = YAML.stringify({ + const validBlueprintDocument = { version: "1.0.0", min_openshell_version: "0.0.106", max_openshell_version: "0.0.106", @@ -389,7 +393,8 @@ describe("packaged Blueprint Runner external target", () => { trust: { ca_file: privateCaPath }, authentication: { credential_file: privateAuthenticationPath }, }, - }); + }; + const validBlueprint = YAML.stringify(validBlueprintDocument); fs.writeFileSync(blueprintFile, validBlueprint); const probePath = path.join(installedPackage, "runtime-probe.mjs"); @@ -509,6 +514,28 @@ describe("packaged Blueprint Runner external target", () => { ); expect(presentStatus.evidence).toEqual({ effects: ["network"] }); + fs.writeFileSync( + blueprintFile, + YAML.stringify({ + ...validBlueprintDocument, + min_openshell_version: "0.0.105", + openshell_target: { + ...validBlueprintDocument.openshell_target, + expected_release: "0.0.105", + }, + }), + ); + const unsupportedReleasePlan = runRunner(["plan"]); + expect(unsupportedReleasePlan.result.status, unsupportedReleasePlan.safeDiagnostics).toBe( + 1, + ); + expect(unsupportedReleasePlan.result.stderr).toContain( + "external OpenShell target expected_release must be 0.0.106", + ); + expect(unsupportedReleasePlan.result.stdout).not.toContain("openshell_target"); + expect(unsupportedReleasePlan.evidence.effects).toEqual([]); + fs.writeFileSync(blueprintFile, validBlueprint); + const plan = runRunner(["plan"]); expect(plan.result.status, plan.safeDiagnostics).toBe(0); expect(plan.evidence.effects).toEqual([]); @@ -541,6 +568,8 @@ describe("packaged Blueprint Runner external target", () => { absentStatus.result.stderr, presentStatus.result.stdout, presentStatus.result.stderr, + unsupportedReleasePlan.result.stdout, + unsupportedReleasePlan.result.stderr, plan.result.stdout, plan.result.stderr, apply.result.stdout, diff --git a/test/repository/source-shape-scanner.test.ts b/test/repository/source-shape-scanner.test.ts index 740eb4710c8..3ab5ad3afe9 100644 --- a/test/repository/source-shape-scanner.test.ts +++ b/test/repository/source-shape-scanner.test.ts @@ -533,6 +533,20 @@ describe("source-shape scanner", () => { expect(cases).toEqual(["mirrors a namespace-loaded workflow"]); }); + it("detects an explicit assertion on the shipped external gateway helper source", () => { + const cases = detectedCaseNames(` + import { expect, it } from "vitest"; + import { readExternalGatewayHealthHelper } from "../tools/e2e/external-gateway-health-workflow-boundary.mts"; + + it("asserts the protected Runner boundary", () => { + const source = readExternalGatewayHealthHelper(); + expect(source).toContain("shellProbe.run"); + }); + `); + + expect(cases).toEqual(["asserts the protected Runner boundary"]); + }); + it("does not taint spawn results when raw config feeds the command", () => { const cases = detectedCaseNames(` import { spawnSync } from "node:child_process"; diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts index ac287251415..e2f3885d130 100644 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ b/tools/e2e/external-gateway-health-workflow-boundary.mts @@ -56,6 +56,10 @@ export function readExternalGatewayHealthWorkflow( return YAML.parse(readFileSync(workflowPath, "utf8")) as ExternalGatewayHealthWorkflow; } +export function readExternalGatewayHealthHelper(helperPath = DEFAULT_LIVE_HELPER_PATH): string { + return readFileSync(helperPath, "utf8"); +} + function findStep(job: WorkflowJob, name: string): WorkflowStep { return job.steps?.find((step) => step.name === name) ?? {}; } @@ -339,7 +343,7 @@ export function validateExternalGatewayHealthHelper(source: string): string[] { export function validateExternalGatewayHealthWorkflow( workflow: ExternalGatewayHealthWorkflow, - liveHelperSource = readFileSync(DEFAULT_LIVE_HELPER_PATH, "utf8"), + liveHelperSource = readExternalGatewayHealthHelper(), ): string[] { const errors: string[] = []; const packageJob = workflow.jobs[PACKAGE_JOB]; @@ -358,6 +362,6 @@ export function validateExternalGatewayHealthWorkflowBoundary( ): string[] { return validateExternalGatewayHealthWorkflow( readExternalGatewayHealthWorkflow(workflowPath), - readFileSync(liveHelperPath, "utf8"), + readExternalGatewayHealthHelper(liveHelperPath), ); } From 5a4f641425b020df5cc271511dc20fabe4793f5d Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 30 Aug 2026 22:24:37 -0700 Subject: [PATCH 06/28] fix(e2e): fail closed external health qualification Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 1 + ci/source-shape-test-budget.json | 15 ++++ scripts/find-source-shape-tests.mts | 13 +-- .../external-gateway-health-process.ts | 42 +++++++++ .../live/external-gateway-health-helpers.ts | 24 ++--- ...l-gateway-health-workflow-boundary.test.ts | 88 +++++++++++++++++-- test/repository/source-shape-scanner.test.ts | 14 +++ ...ernal-gateway-health-workflow-boundary.mts | 1 + 8 files changed, 171 insertions(+), 27 deletions(-) create mode 100644 test/e2e/fixtures/external-gateway-health-process.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a80b3812e3a..d39976d31cb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -3419,6 +3419,7 @@ jobs: NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106" + NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index aebff976f05..00292940d22 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -41,6 +41,11 @@ "test": "should return actionable unknown target error", "category": "compatibility" }, + { + "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", + "test": "accepts the checked-in trusted package and live-test contract", + "category": "security" + }, { "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", "test": "rejects a helper that bypasses the exact Runner or weakens its live boundary", @@ -56,6 +61,16 @@ "test": "rejects a Runner invocation that bypasses the redacted shell fixture", "category": "security" }, + { + "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", + "test": "rejects package credentials or untrusted candidate execution in the package job", + "category": "security" + }, + { + "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", + "test": "rejects credential exposure and candidate or artifact substitution in the live job", + "category": "security" + }, { "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", "test": "keeps every candidate on standard runners when $name (#7145)", diff --git a/scripts/find-source-shape-tests.mts b/scripts/find-source-shape-tests.mts index b90372dc34e..18b0291b153 100755 --- a/scripts/find-source-shape-tests.mts +++ b/scripts/find-source-shape-tests.mts @@ -301,8 +301,8 @@ function isDynamicFunctionConstructorFactory(expression: ts.Expression): boolean const candidate = prototypeCall.arguments[0]; return Boolean( candidate && - (ts.isArrowFunction(candidate) || ts.isFunctionExpression(candidate)) && - candidate.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword), + (ts.isArrowFunction(candidate) || ts.isFunctionExpression(candidate)) && + candidate.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword), ); } @@ -590,6 +590,7 @@ function collectDeclarativeImports(imports: readonly ImportBinding[]): Map> = { "tools/e2e/external-gateway-health-workflow-boundary": [ "readExternalGatewayHealthHelper", + "readExternalGatewayHealthWorkflow", ], "test/helpers/e2e-workflow-contract": ["readWorkflow", "readYaml"], "test/e2e/registry/registry": ["getTarget", "listTargets", "requireTargets"], @@ -1494,9 +1495,9 @@ function assertionFromAssertCall( const firstRoot = firstArgument ? rootIdentifier(firstArgument) : null; const firstIsBehavior = Boolean( firstArgument && - (isProductionBehaviorDerivation(firstArgument, productionConsumerNames) || - isExecutionResultDerivation(sourceFile, firstArgument) || - (firstRoot && executionResultVars.has(firstRoot))), + (isProductionBehaviorDerivation(firstArgument, productionConsumerNames) || + isExecutionResultDerivation(sourceFile, firstArgument) || + (firstRoot && executionResultVars.has(firstRoot))), ); const argumentCount = ASSERT_TWO_ARGUMENT_MATCHERS.has(method) && !firstIsBehavior ? 2 : 1; for (const argument of node.arguments.slice(0, argumentCount)) { @@ -2136,7 +2137,7 @@ function isDirectInvocation(): boolean { const invoked = process.argv[1]; return Boolean( invoked && - (import.meta.url === `file://${invoked}` || invoked.endsWith("find-source-shape-tests.mts")), + (import.meta.url === `file://${invoked}` || invoked.endsWith("find-source-shape-tests.mts")), ); } diff --git a/test/e2e/fixtures/external-gateway-health-process.ts b/test/e2e/fixtures/external-gateway-health-process.ts new file mode 100644 index 00000000000..44d7fa0dcdd --- /dev/null +++ b/test/e2e/fixtures/external-gateway-health-process.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ChildProcess } from "node:child_process"; + +const GATEWAY_STOP_TIMEOUT_MS = 2_000; + +export function externalGatewayHealthProcessStopped(gateway: ChildProcess): boolean { + return gateway.exitCode !== null || gateway.signalCode !== null; +} + +function waitForGatewayStop(gateway: ChildProcess): Promise { + if (externalGatewayHealthProcessStopped(gateway)) return Promise.resolve(true); + return new Promise((resolve) => { + let settled = false; + const finish = (stopped: boolean): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + gateway.off("exit", onExit); + resolve(stopped); + }; + const onExit = (): void => finish(true); + const timer = setTimeout( + () => finish(externalGatewayHealthProcessStopped(gateway)), + GATEWAY_STOP_TIMEOUT_MS, + ); + timer.unref(); + gateway.once("exit", onExit); + if (externalGatewayHealthProcessStopped(gateway)) finish(true); + }); +} + +export async function stopExternalGatewayHealthGateway(gateway: ChildProcess): Promise { + if (externalGatewayHealthProcessStopped(gateway)) return; + gateway.kill("SIGTERM"); + if (await waitForGatewayStop(gateway)) return; + gateway.kill("SIGKILL"); + if (!(await waitForGatewayStop(gateway))) { + throw new Error("external gateway health gateway did not stop after SIGKILL"); + } +} diff --git a/test/e2e/live/external-gateway-health-helpers.ts b/test/e2e/live/external-gateway-health-helpers.ts index eb43778758c..04d0ca9adb6 100644 --- a/test/e2e/live/external-gateway-health-helpers.ts +++ b/test/e2e/live/external-gateway-health-helpers.ts @@ -12,6 +12,10 @@ import { getDockerDriverGatewayLocalTlsBundle } from "../../../dist/lib/onboard/ import type { ArtifactSink } from "../fixtures/artifacts.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import { expect } from "../fixtures/e2e-test.ts"; +import { + externalGatewayHealthProcessStopped, + stopExternalGatewayHealthGateway, +} from "../fixtures/external-gateway-health-process.ts"; import { OPENSHELL_V0106_QUALIFICATION } from "../fixtures/openshell-v0106-qualification.ts"; import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; @@ -74,10 +78,6 @@ function pickPort(host: string): Promise { }); } -function delay(milliseconds: number): Promise { - return new Promise((resolve) => setTimeout(resolve, milliseconds)); -} - function externalHostAddress(): string { for (const addresses of Object.values(os.networkInterfaces())) { for (const address of addresses ?? []) { @@ -87,16 +87,6 @@ function externalHostAddress(): string { throw new Error("a non-loopback IPv4 address is required for external gateway health"); } -async function stopGateway(gateway: ChildProcess): Promise { - if (gateway.exitCode !== null) return; - gateway.kill("SIGTERM"); - for (let attempt = 0; attempt < 20; attempt += 1) { - if (gateway.exitCode !== null) return; - await delay(100); - } - gateway.kill("SIGKILL"); -} - function parseRunnerStatus(output: string): Record { const jsonStart = output.indexOf("{"); if (jsonStart < 0) throw new Error("Blueprint Runner status did not produce JSON"); @@ -144,7 +134,7 @@ async function waitForGatewayPort(options: { await options.artifacts.writeJson("external-gateway-readiness-retry.json", evidence); }, run: () => - options.gateway.exitCode === null + !externalGatewayHealthProcessStopped(options.gateway) ? observeGatewayPort(options.address, options.port) : Promise.resolve({ errorCode: "gateway-exited", ready: false }), classify: (value, error) => { @@ -286,7 +276,9 @@ export async function runExternalGatewayHealthScenario({ gateway.stderr?.on("data", (chunk: Buffer) => { gatewayOutput += chunk.toString("utf8"); }); - cleanup.add("stop external gateway health gateway", () => stopGateway(gateway)); + cleanup.add("stop external gateway health gateway", () => + stopExternalGatewayHealthGateway(gateway), + ); try { const blueprintRoot = path.join(stateDir, "blueprint"); diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index 1335e1c9b1a..e3a50303409 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -1,7 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { EventEmitter } from "node:events"; +import type { ChildProcess } from "node:child_process"; + +import { describe, expect, it, vi } from "vitest"; + +import { stopExternalGatewayHealthGateway } from "../fixtures/external-gateway-health-process.ts"; import { readExternalGatewayHealthHelper, @@ -12,7 +17,12 @@ import { } from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; describe("external gateway health workflow boundary", () => { + // source-shape-contract: security -- The checked-in workflow and helper must retain the trusted package and Runner execution boundaries. it("accepts the checked-in trusted package and live-test contract", () => { + const workflow = readExternalGatewayHealthWorkflow(); + const helperSource = readExternalGatewayHealthHelper(); + expect(workflow.jobs).toHaveProperty("package-openshell-sdk"); + expect(helperSource).toContain("const BLUEPRINT_RUNNER = path.join("); expect(validateExternalGatewayHealthWorkflowBoundary()).toEqual([]); }); @@ -45,10 +55,7 @@ describe("external gateway health workflow boundary", () => { it("rejects a different module-relative Runner artifact", () => { const helperSource = readExternalGatewayHealthHelper(); expect(helperSource).toContain(' "blueprint-runner.js",'); - const source = helperSource.replace( - ' "blueprint-runner.js",', - ' "other-runner.js",', - ); + const source = helperSource.replace(' "blueprint-runner.js",', ' "other-runner.js",'); expect(validateExternalGatewayHealthHelper(source)).toContain( "external gateway health helper must run exact Blueprint Runner external status", @@ -69,8 +76,10 @@ describe("external gateway health workflow boundary", () => { ); }); + // source-shape-contract: security -- The trusted package job must not give package credentials or candidate code control of the SDK archive. it("rejects package credentials or untrusted candidate execution in the package job", () => { const workflow = readExternalGatewayHealthWorkflow(); + expect(workflow.jobs).toHaveProperty("package-openshell-sdk"); const job = workflow.jobs["package-openshell-sdk"]; job.if = "${{ always() }}"; job.permissions = { contents: "write", packages: "write" }; @@ -91,10 +100,13 @@ describe("external gateway health workflow boundary", () => { ); }); + // source-shape-contract: security -- The live job must run the exact candidate with the reviewed SDK archive and no package credential. it("rejects credential exposure and candidate or artifact substitution in the live job", () => { const workflow = readExternalGatewayHealthWorkflow(); + expect(workflow.jobs).toHaveProperty("external-gateway-health"); const job = workflow.jobs["external-gateway-health"]; job.needs = "generate-matrix"; + delete job.env!.NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST; job.env = { ...job.env, GITHUB_TOKEN: "${{ github.token }}" }; const checkout = job.steps!.find((step) => step.uses?.startsWith("actions/checkout@"))!; checkout.with!.ref = "main"; @@ -112,6 +124,7 @@ describe("external gateway health workflow boundary", () => { expect(validateExternalGatewayHealthWorkflow(workflow)).toEqual( expect.arrayContaining([ "external-gateway-health must wait for the candidate CLI and reviewed SDK archive", + "external-gateway-health must retain NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST=1", "external-gateway-health must not expose GITHUB_TOKEN at job scope", "external-gateway-health must use the exact candidate checkout without persisted credentials", "external-gateway-health must download only this run's reviewed SDK archive", @@ -121,4 +134,69 @@ describe("external gateway health workflow boundary", () => { ]), ); }); + + type FakeGateway = EventEmitter & { + exitCode: number | null; + signalCode: NodeJS.Signals | null; + kill: ChildProcess["kill"]; + }; + + function gatewayWithForcedKill(onForcedKill: (gateway: FakeGateway) => void): { + gateway: ChildProcess; + kill: ReturnType>; + } { + const gateway = new EventEmitter() as FakeGateway; + gateway.exitCode = null; + gateway.signalCode = null; + const kill = vi.fn(); + kill.mockReturnValueOnce(true); + kill.mockImplementationOnce(() => { + onForcedKill(gateway); + return true; + }); + gateway.kill = kill; + return { gateway: gateway as ChildProcess, kill }; + } + + it("waits for forced gateway exit before cleanup completes (#9872)", async () => { + vi.useFakeTimers(); + try { + const { gateway, kill } = gatewayWithForcedKill((fakeGateway) => { + setTimeout(() => { + fakeGateway.signalCode = "SIGKILL"; + fakeGateway.emit("exit", null, "SIGKILL"); + }, 50); + }); + let completed = false; + const cleanup = stopExternalGatewayHealthGateway(gateway).then(() => { + completed = true; + }); + + await vi.advanceTimersByTimeAsync(2_000); + expect(kill).toHaveBeenNthCalledWith(1, "SIGTERM"); + expect(kill).toHaveBeenNthCalledWith(2, "SIGKILL"); + expect(completed).toBe(false); + + await vi.advanceTimersByTimeAsync(50); + await cleanup; + expect(completed).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it("reports a gateway that remains active after forced cleanup (#9872)", async () => { + vi.useFakeTimers(); + try { + const { gateway } = gatewayWithForcedKill(() => undefined); + const cleanup = expect(stopExternalGatewayHealthGateway(gateway)).rejects.toThrow( + "external gateway health gateway did not stop after SIGKILL", + ); + + await vi.advanceTimersByTimeAsync(4_000); + await cleanup; + } finally { + vi.useRealTimers(); + } + }); }); diff --git a/test/repository/source-shape-scanner.test.ts b/test/repository/source-shape-scanner.test.ts index 3ab5ad3afe9..f6cda57f4dd 100644 --- a/test/repository/source-shape-scanner.test.ts +++ b/test/repository/source-shape-scanner.test.ts @@ -547,6 +547,20 @@ describe("source-shape scanner", () => { expect(cases).toEqual(["asserts the protected Runner boundary"]); }); + it("detects an explicit assertion on the shipped external gateway workflow", () => { + const cases = detectedCaseNames(` + import { expect, it } from "vitest"; + import { readExternalGatewayHealthWorkflow } from "../tools/e2e/external-gateway-health-workflow-boundary.mts"; + + it("asserts the protected package job", () => { + const workflow = readExternalGatewayHealthWorkflow(); + expect(workflow.jobs).toHaveProperty("package-openshell-sdk"); + }); + `); + + expect(cases).toEqual(["asserts the protected package job"]); + }); + it("does not taint spawn results when raw config feeds the command", () => { const cases = detectedCaseNames(` import { spawnSync } from "node:child_process"; diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts index e2f3885d130..72d038cfa47 100644 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ b/tools/e2e/external-gateway-health-workflow-boundary.mts @@ -197,6 +197,7 @@ function validateHealthJob(errors: string[], job: WorkflowJob): void { E2E_OBSERVABLE_OUTCOME: "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA", E2E_TARGET_ID: HEALTH_JOB, + NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106", NEMOCLAW_RUN_LIVE_E2E: "1", From f5b57e1aff5b0b5e7ca98b12a82470b07d19bcc3 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 30 Aug 2026 22:51:47 -0700 Subject: [PATCH 07/28] fix(blueprint): align external target boundaries Signed-off-by: Apurv Kumaria --- .../blueprint/runner-external-target.test.ts | 52 +++++++++++---- nemoclaw/src/blueprint/runner.ts | 63 +++++++++---------- nemoclaw/tsconfig.runner.json | 5 +- .../blueprint-external-target-plan.test.ts | 12 +++- 4 files changed, 79 insertions(+), 53 deletions(-) diff --git a/nemoclaw/src/blueprint/runner-external-target.test.ts b/nemoclaw/src/blueprint/runner-external-target.test.ts index bd01b8339cc..dd9fcecb2d0 100644 --- a/nemoclaw/src/blueprint/runner-external-target.test.ts +++ b/nemoclaw/src/blueprint/runner-external-target.test.ts @@ -58,7 +58,8 @@ vi.mock("./ssrf.js", async (importOriginal) => { const { validateEndpointUrl } = await import("./ssrf.js"); const mockedValidateEndpoint = vi.mocked(validateEndpointUrl); -const { actionExternalOpenShellTargetPlan, main } = await import("./runner.js"); +const { actionExternalOpenShellTargetPlan, actionExternalOpenShellTargetStatus, main } = + await import("./runner.js"); const EXTERNAL_CA_FILE = "/var/run/openshell-target/private-ca.pem"; const EXTERNAL_AUTHENTICATION_FILE = "/var/run/openshell-target/private-authentication"; @@ -120,13 +121,25 @@ describe("Blueprint Runner external OpenShell target", () => { vi.restoreAllMocks(); }); - it("requires an external target and a complete OpenShell version range", () => { + it("requires an external target and a complete OpenShell version range", async () => { expect(() => actionExternalOpenShellTargetPlan({})).toThrow( /does not declare an external OpenShell target/, ); - expect(() => - actionExternalOpenShellTargetPlan({ openshell_target: {} }), - ).toThrow(/requires blueprint min_openshell_version and max_openshell_version/); + expect(() => actionExternalOpenShellTargetPlan({ openshell_target: {} })).toThrow( + /requires blueprint min_openshell_version and max_openshell_version/, + ); + await expect(actionExternalOpenShellTargetStatus({}, gatewayHealthObserver)).rejects.toThrow( + /does not declare an external OpenShell target/, + ); + await expect( + actionExternalOpenShellTargetStatus({ openshell_target: {} }, gatewayHealthObserver), + ).rejects.toThrow(/requires blueprint min_openshell_version and max_openshell_version/); + + expect( + externalTargetBoundaryMocks.buildSanitizedExternalOpenShellTargetPlan, + ).not.toHaveBeenCalled(); + expect(externalTargetBoundaryMocks.withExternalOpenShellTargetCa).not.toHaveBeenCalled(); + expect(observeHealth).not.toHaveBeenCalled(); }); it("emits only the sanitized plan without subprocess or network calls (#9872)", async () => { @@ -272,24 +285,39 @@ describe("Blueprint Runner external OpenShell target", () => { }); it.each([ - ["components", { components: { sandbox: { name: "managed" } } }], - ["profiles", { profiles: ["default"] }], - ["min_openclaw_version", { min_openclaw_version: "2026.8.0" }], + ["components", "planning", ["plan"], { components: { sandbox: { name: "managed" } } }], + [ + "components", + "status", + ["status", "--external-target"], + { components: { sandbox: { name: "managed" } } }, + ], + ["profiles", "planning", ["plan"], { profiles: ["default"] }], + ["profiles", "status", ["status", "--external-target"], { profiles: ["default"] }], + ["min_openclaw_version", "planning", ["plan"], { min_openclaw_version: "2026.8.0" }], + [ + "min_openclaw_version", + "status", + ["status", "--external-target"], + { min_openclaw_version: "2026.8.0" }, + ], ])( - "rejects the managed-only blueprint field %s before any effect (#9872)", - async (field, value) => { + "rejects the managed-only blueprint field %s from external target %s before any effect (#9872)", + async (field, action, argv, value) => { addFile("blueprint.yaml", YAML.stringify({ ...externalTargetBlueprint(), ...value })); - await expect(runMain(["plan"])).rejects.toThrow( - `External OpenShell target planning does not accept '${field}'.`, + await expect(runMain(argv)).rejects.toThrow( + `External OpenShell target ${action} does not accept '${field}'.`, ); expect(mockExeca).not.toHaveBeenCalled(); expect(mockedValidateEndpoint).not.toHaveBeenCalled(); + expect(observeHealth).not.toHaveBeenCalled(); expect(stdoutCapture.text()).not.toContain("RUN_ID:"); expect( externalTargetBoundaryMocks.buildSanitizedExternalOpenShellTargetPlan, ).not.toHaveBeenCalled(); + expect(externalTargetBoundaryMocks.withExternalOpenShellTargetCa).not.toHaveBeenCalled(); }, ); diff --git a/nemoclaw/src/blueprint/runner.ts b/nemoclaw/src/blueprint/runner.ts index ce541f625c1..a8c4952612d 100644 --- a/nemoclaw/src/blueprint/runner.ts +++ b/nemoclaw/src/blueprint/runner.ts @@ -40,7 +40,10 @@ import * as importedOpenShellExternalTargetBoundary from "../shared/openshell-ex import * as importedOpenShellObservationBoundary from "../shared/openshell-observation-boundary.cjs"; import * as importedOpenShellPolicyBoundary from "../shared/openshell-policy-boundary.cjs"; import * as importedSandboxName from "../shared/sandbox-name.cjs"; -import type { SanitizedExternalOpenShellTargetPlan } from "../shared/openshell-external-target-boundary.cjs"; +import type { + OpenShellCompatibilityRange, + SanitizedExternalOpenShellTargetPlan, +} from "../shared/openshell-external-target-boundary.cjs"; import type { ExternalOpenShellGatewayStatus, OpenShellGatewayHealthObserver, @@ -606,10 +609,7 @@ function mergePolicyAdditions(currentPolicyRaw: string, additions: PolicyAdditio } function blueprintBasePoliciesMatch(left: string, right: string): boolean { - return isDeepStrictEqual( - parseOpenShellPolicy(left).policy, - parseOpenShellPolicy(right).policy, - ); + return isDeepStrictEqual(parseOpenShellPolicy(left).policy, parseOpenShellPolicy(right).policy); } function assertNoConflictingBlueprintPolicyChange( @@ -1377,29 +1377,11 @@ export function actionExternalOpenShellTargetPlan( blueprint: Blueprint, options?: { dryRun?: boolean }, ): ExternalOpenShellTargetRunPlan { - if (blueprint.openshell_target === undefined) { - throw new Error("blueprint does not declare an external OpenShell target"); - } - if ( - blueprint.min_openshell_version === undefined || - blueprint.max_openshell_version === undefined - ) { - throw new Error( - "External OpenShell target planning requires blueprint min_openshell_version and max_openshell_version.", - ); - } - for (const field of ["components", "profiles", "min_openclaw_version"] as const) { - if (blueprint[field] !== undefined) { - throw new Error(`External OpenShell target planning does not accept '${field}'.`); - } - } + const { target, compatibility } = validateExternalOpenShellTargetBlueprint(blueprint, "planning"); const rid = emitRunId(); progress(10, "Validating external OpenShell target"); - const targetPlan = buildSanitizedExternalOpenShellTargetPlan(blueprint.openshell_target, { - minVersion: blueprint.min_openshell_version, - maxVersion: blueprint.max_openshell_version, - }); + const targetPlan = buildSanitizedExternalOpenShellTargetPlan(target, compatibility); const plan: ExternalOpenShellTargetRunPlan = { run_id: rid, openshell_target: targetPlan, @@ -1410,10 +1392,10 @@ export function actionExternalOpenShellTargetPlan( return plan; } -export async function actionExternalOpenShellTargetStatus( +function validateExternalOpenShellTargetBlueprint( blueprint: Blueprint, - observer: OpenShellGatewayHealthObserver, -): Promise { + action: "planning" | "status", +): Readonly<{ target: unknown; compatibility: OpenShellCompatibilityRange }> { if (blueprint.openshell_target === undefined) { throw new Error("blueprint does not declare an external OpenShell target"); } @@ -1422,23 +1404,34 @@ export async function actionExternalOpenShellTargetStatus( blueprint.max_openshell_version === undefined ) { throw new Error( - "External OpenShell target status requires blueprint min_openshell_version and max_openshell_version.", + `External OpenShell target ${action} requires blueprint min_openshell_version and max_openshell_version.`, ); } for (const field of ["components", "profiles", "min_openclaw_version"] as const) { if (blueprint[field] !== undefined) { - throw new Error(`External OpenShell target status does not accept '${field}'.`); + throw new Error(`External OpenShell target ${action} does not accept '${field}'.`); } } + return { + target: blueprint.openshell_target, + compatibility: { + minVersion: blueprint.min_openshell_version, + maxVersion: blueprint.max_openshell_version, + }, + }; +} + +export async function actionExternalOpenShellTargetStatus( + blueprint: Blueprint, + observer: OpenShellGatewayHealthObserver, +): Promise { + const { target, compatibility } = validateExternalOpenShellTargetBlueprint(blueprint, "status"); const rid = emitRunId(); progress(10, "Validating external OpenShell target"); const observation = await withExternalOpenShellTargetCa( - blueprint.openshell_target, - { - minVersion: blueprint.min_openshell_version, - maxVersion: blueprint.max_openshell_version, - }, + target, + compatibility, async (target, caContents) => { const observed = await observeExternalOpenShellGatewayHealth(observer, { target, diff --git a/nemoclaw/tsconfig.runner.json b/nemoclaw/tsconfig.runner.json index bc30a633e0e..8b6870806d4 100644 --- a/nemoclaw/tsconfig.runner.json +++ b/nemoclaw/tsconfig.runner.json @@ -7,9 +7,6 @@ "rootDir": "src", "sourceMap": false }, - "include": [ - "src/blueprint/runner.ts", - "src/shared/openshell-gateway-health-sdk.ts" - ], + "include": ["src/blueprint/runner.ts"], "exclude": ["node_modules", "dist", "runner-dist", "src/**/*.test.ts"] } diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 2f063264154..ba110bea7d9 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -318,7 +318,14 @@ describe("packaged Blueprint Runner external target", () => { "openshell", "sandbox-observer.js", ); - const installedSdkObserver = path.join( + const installedSharedSdkObserver = path.join( + installedPackage, + "nemoclaw", + "dist", + "shared", + "openshell-gateway-health-sdk.js", + ); + const installedRuntimeSdkObserver = path.join( installedPackage, "dist", "nemoclaw", @@ -333,7 +340,8 @@ describe("packaged Blueprint Runner external target", () => { ); expect(fs.existsSync(installedRunner)).toBe(true); expect(fs.existsSync(installedRootObserver)).toBe(true); - expect(fs.existsSync(installedSdkObserver)).toBe(true); + expect(fs.existsSync(installedSharedSdkObserver)).toBe(true); + expect(fs.existsSync(installedRuntimeSdkObserver)).toBe(false); expect(fs.existsSync(path.join(installedPackage, "nemoclaw", "src"))).toBe(false); expect(fs.existsSync(installedRuntimeRunner)).toBe(true); expect( From ba6173405ee37cfca45a3afe12a30994805848d0 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 30 Aug 2026 23:16:30 -0700 Subject: [PATCH 08/28] docs(schema): describe external target status Signed-off-by: Apurv Kumaria --- schemas/blueprint.schema.json | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/schemas/blueprint.schema.json b/schemas/blueprint.schema.json index efb14aa862a..f4df45028c9 100644 --- a/schemas/blueprint.schema.json +++ b/schemas/blueprint.schema.json @@ -91,8 +91,16 @@ "type": "object", "required": ["cpu", "memory"], "properties": { - "cpu": { "type": "string", "minLength": 1, "pattern": "^([1-9]\\d?%|100%|\\d+(\\.\\d+)?[mKMGTPE]?i?)$" }, - "memory": { "type": "string", "minLength": 1, "pattern": "^([1-9]\\d?%|100%|\\d+(\\.\\d+)?[mKMGTPE]?i?)$" } + "cpu": { + "type": "string", + "minLength": 1, + "pattern": "^([1-9]\\d?%|100%|\\d+(\\.\\d+)?[mKMGTPE]?i?)$" + }, + "memory": { + "type": "string", + "minLength": 1, + "pattern": "^([1-9]\\d?%|100%|\\d+(\\.\\d+)?[mKMGTPE]?i?)$" + } }, "additionalProperties": false } @@ -164,7 +172,7 @@ "$defs": { "externalOpenShellTarget": { "type": "object", - "description": "Explicit externally managed OpenShell gateway target for target-only planning. File references use POSIX-rooted absolute paths. This contract does not establish a network connection, TLS trust, authentication, remote readiness, machine-authentication compatibility, or Kubernetes qualification.", + "description": "Explicit externally managed OpenShell gateway target for target-only planning and credential-free public health status. File references use POSIX-rooted absolute paths. Planning validates and sanitizes target inputs without connecting. Status checks authentication file metadata, then uses the explicit HTTPS endpoint and validated CA bundle for one public health request. Status does not read authentication file contents or authenticate. This contract does not establish remote readiness or support for machine authentication or Kubernetes.", "required": [ "endpoint", "workspace", @@ -184,12 +192,12 @@ "type": "string", "pattern": "^(?!.*--)[a-z]([a-z0-9-]*[a-z0-9])?$", "maxLength": 19, - "description": "Explicit OpenShell workspace name for the target-only plan." + "description": "Explicit OpenShell workspace name included in the sanitized plan and status identity. Public health does not verify or access the workspace." }, "expected_release": { "type": "string", "pattern": "^[0-9]{1,15}\\.[0-9]{1,15}\\.[0-9]{1,15}$", - "description": "Exact OpenShell release expected by the plan. The runner requires safe numeric components and validates this value against min_openshell_version and max_openshell_version." + "description": "Exact OpenShell release required by planning and public health status. The runner requires safe numeric components and validates this value against min_openshell_version and max_openshell_version. Status succeeds only when public health reports this release." }, "lifecycle": { "const": "external", @@ -204,7 +212,7 @@ "type": "string", "minLength": 1, "pattern": "^/", - "description": "POSIX-rooted absolute path to a regular PEM CA bundle no larger than 1 MiB. Planning parses each certificate, requires the CA flag, and returns a fingerprint. It does not connect to the endpoint or establish TLS trust. Symbolic-link paths are not accepted." + "description": "POSIX-rooted absolute path to a regular PEM CA bundle no larger than 1 MiB. Planning validates the PEM CA bundle and returns its fingerprint without connecting. Status supplies the validated bundle to the official OpenShell SDK for the credential-free public health request. Symbolic-link paths are not accepted." } } }, @@ -212,13 +220,13 @@ "type": "object", "required": ["credential_file"], "additionalProperties": false, - "description": "File-backed authentication reference for target-only planning. Planning checks file metadata only; it does not read the contents, authenticate, or select or validate a machine-authentication protocol.", + "description": "File-backed authentication reference required for target-only planning and status. Both operations check file metadata only. Neither reads its contents, authenticates, or selects or validates a machine-authentication protocol.", "properties": { "credential_file": { "type": "string", "minLength": 1, "pattern": "^/", - "description": "POSIX-rooted absolute path to a non-empty regular authentication file no larger than 1 MiB. Symbolic-link paths are not accepted. Planning checks metadata without reading the file and does not return the path." + "description": "POSIX-rooted absolute path to a non-empty regular authentication file no larger than 1 MiB. Symbolic-link paths are not accepted. The planning and public health status operations inspect only file metadata. Neither operation reads the file contents or returns the path." } } } From 259d4741ef245340fe27735d95d8281e21999a46 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 13:55:04 -0700 Subject: [PATCH 09/28] test(blueprint): replace external status shape checks Signed-off-by: Apurv Kumaria --- ci/source-shape-test-budget.json | 40 ------ .../pr-workflow-contract.test.ts | 7 - ...l-gateway-health-workflow-boundary.test.ts | 126 ------------------ .../blueprint-external-target-plan.test.ts | 7 - 4 files changed, 180 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 00292940d22..63e0642100f 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -41,36 +41,6 @@ "test": "should return actionable unknown target error", "category": "compatibility" }, - { - "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", - "test": "accepts the checked-in trusted package and live-test contract", - "category": "security" - }, - { - "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", - "test": "rejects a helper that bypasses the exact Runner or weakens its live boundary", - "category": "security" - }, - { - "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", - "test": "rejects a different module-relative Runner artifact", - "category": "security" - }, - { - "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", - "test": "rejects a Runner invocation that bypasses the redacted shell fixture", - "category": "security" - }, - { - "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", - "test": "rejects package credentials or untrusted candidate execution in the package job", - "category": "security" - }, - { - "file": "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", - "test": "rejects credential exposure and candidate or artifact substitution in the live job", - "category": "security" - }, { "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", "test": "keeps every candidate on standard runners when $name (#7145)", @@ -146,11 +116,6 @@ "test": "rejects an isolated unreviewed Deep Agents Code Node base pin", "category": "security" }, - { - "file": "test/package-contract/blueprint-external-target-plan.test.ts", - "test": "executes the installed package command and fails before effects when the SDK is absent (#9872)", - "category": "compatibility" - }, { "file": "test/agents/openclaw/openclaw-2026-6-npm-remediation.test.ts", "test": "rebuilds a guarded core archive with the patched fs-safe package bundled", @@ -226,11 +191,6 @@ "test": "provides the package token only to trusted main dependency installation", "category": "security" }, - { - "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", - "test": "reuses the trusted dependency cache for offline package contracts", - "category": "security" - }, { "file": "test/automation/pull-requests/pr-workflow-contract.test.ts", "test": "passes only the base-packaged SDK archive to pull request dependency jobs", diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index ab47706d9b8..dfab93ea1ee 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -534,13 +534,6 @@ describe("pull request and main workflow contracts", () => { ); }); - // source-shape-contract: security -- Offline package checks must use only the cache populated by the trusted dependency installer - it("reuses the trusted dependency cache for offline package contracts", () => { - expect( - requiredStep(sharedActions.buildTypecheck, "Verify compiled package contracts").env, - ).toEqual({ NPM_CONFIG_CACHE: "${{ runner.temp }}/npm" }); - }); - // source-shape-contract: security -- The PR workflow must select an exact base-controlled package run before publishing its archive internally it("passes only the base-packaged SDK archive to pull request dependency jobs", () => { const packageJob = prWorkflow.jobs["openshell-sdk-package"]; diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index e3a50303409..c8435b13885 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -8,133 +8,7 @@ import { describe, expect, it, vi } from "vitest"; import { stopExternalGatewayHealthGateway } from "../fixtures/external-gateway-health-process.ts"; -import { - readExternalGatewayHealthHelper, - readExternalGatewayHealthWorkflow, - validateExternalGatewayHealthHelper, - validateExternalGatewayHealthWorkflow, - validateExternalGatewayHealthWorkflowBoundary, -} from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; - describe("external gateway health workflow boundary", () => { - // source-shape-contract: security -- The checked-in workflow and helper must retain the trusted package and Runner execution boundaries. - it("accepts the checked-in trusted package and live-test contract", () => { - const workflow = readExternalGatewayHealthWorkflow(); - const helperSource = readExternalGatewayHealthHelper(); - expect(workflow.jobs).toHaveProperty("package-openshell-sdk"); - expect(helperSource).toContain("const BLUEPRINT_RUNNER = path.join("); - expect(validateExternalGatewayHealthWorkflowBoundary()).toEqual([]); - }); - - // source-shape-contract: security -- The live helper must keep SDK access behind the packaged Runner, bounded retry, and redacted evidence. - it("rejects a helper that bypasses the exact Runner or weakens its live boundary", () => { - const helperSource = readExternalGatewayHealthHelper(); - expect(helperSource).toContain("const BLUEPRINT_RUNNER = path.join("); - const source = helperSource - .replace('"--external-target"', '"--managed-target"') - .replace("const address = externalHostAddress();", 'const address = "127.0.0.1";') - .replace("maxAttempts: 10,", "maxAttempts: 100,") - .replace( - 'runner: "dist/lib/blueprint-runner.js"', - 'runner: "test/e2e/live/external-gateway-health-helpers.ts"', - ) - .replace("artifacts.addRedactionValues([stateDir]);", "") - .concat('\nimport "@nvidia/openshell-sdk";\n'); - - expect(validateExternalGatewayHealthHelper(source)).toEqual([ - "external gateway health helper must run exact Blueprint Runner external status", - "external gateway health helper must bind the gateway certificate to a non-loopback address", - "external gateway health helper must retain the bounded readiness retry", - "external gateway health helper must record the exact Runner artifact identity", - "external gateway health helper must redact its private fixture path", - "external gateway health helper must not bypass the Runner with a direct SDK import", - ]); - }); - - // source-shape-contract: security -- The live helper must execute the candidate Runner module before its health evidence is trusted. - it("rejects a different module-relative Runner artifact", () => { - const helperSource = readExternalGatewayHealthHelper(); - expect(helperSource).toContain(' "blueprint-runner.js",'); - const source = helperSource.replace(' "blueprint-runner.js",', ' "other-runner.js",'); - - expect(validateExternalGatewayHealthHelper(source)).toContain( - "external gateway health helper must run exact Blueprint Runner external status", - ); - }); - - // source-shape-contract: security -- The live helper must keep Runner output behind the bounded redacted shell boundary. - it("rejects a Runner invocation that bypasses the redacted shell fixture", () => { - const helperSource = readExternalGatewayHealthHelper(); - expect(helperSource).toContain("const result = await shellProbe.run("); - const source = helperSource.replace( - "const result = await shellProbe.run(", - "const result = spawnSync(", - ); - - expect(validateExternalGatewayHealthHelper(source)).toContain( - "external gateway health helper must run exact Blueprint Runner external status", - ); - }); - - // source-shape-contract: security -- The trusted package job must not give package credentials or candidate code control of the SDK archive. - it("rejects package credentials or untrusted candidate execution in the package job", () => { - const workflow = readExternalGatewayHealthWorkflow(); - expect(workflow.jobs).toHaveProperty("package-openshell-sdk"); - const job = workflow.jobs["package-openshell-sdk"]; - job.if = "${{ always() }}"; - job.permissions = { contents: "write", packages: "write" }; - const checkout = job.steps!.find((step) => step.uses?.startsWith("actions/checkout@"))!; - checkout.with!.ref = "${{ inputs.checkout_sha }}"; - const download = job.steps!.find( - (step) => step.name === "Download and verify exact OpenShell SDK package", - )!; - download.env!.NODE_AUTH_TOKEN = "${{ secrets.PACKAGE_TOKEN }}"; - - expect(validateExternalGatewayHealthWorkflow(workflow)).toEqual( - expect.arrayContaining([ - "package-openshell-sdk must run only for the explicit external health selector", - "package-openshell-sdk must retain its bounded package-read trust boundary", - "package-openshell-sdk must execute only the trusted sparse package verifier checkout", - "package-openshell-sdk must scope its package credential to the reviewed downloader", - ]), - ); - }); - - // source-shape-contract: security -- The live job must run the exact candidate with the reviewed SDK archive and no package credential. - it("rejects credential exposure and candidate or artifact substitution in the live job", () => { - const workflow = readExternalGatewayHealthWorkflow(); - expect(workflow.jobs).toHaveProperty("external-gateway-health"); - const job = workflow.jobs["external-gateway-health"]; - job.needs = "generate-matrix"; - delete job.env!.NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST; - job.env = { ...job.env, GITHUB_TOKEN: "${{ github.token }}" }; - const checkout = job.steps!.find((step) => step.uses?.startsWith("actions/checkout@"))!; - checkout.with!.ref = "main"; - const download = job.steps!.find( - (step) => step.name === "Download reviewed OpenShell SDK archive", - )!; - download.with!.name = "unreviewed-sdk"; - const install = job.steps!.find( - (step) => step.name === "Install reviewed OpenShell SDK archive without package credentials", - )!; - install.run = "npm install @nvidia/openshell-sdk@latest"; - const run = job.steps!.find((step) => step.name === "Run external gateway health live test")!; - run.env = { NODE_AUTH_TOKEN: "${{ secrets.PACKAGE_TOKEN }}" }; - - expect(validateExternalGatewayHealthWorkflow(workflow)).toEqual( - expect.arrayContaining([ - "external-gateway-health must wait for the candidate CLI and reviewed SDK archive", - "external-gateway-health must retain NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST=1", - "external-gateway-health must not expose GITHUB_TOKEN at job scope", - "external-gateway-health must use the exact candidate checkout without persisted credentials", - "external-gateway-health must download only this run's reviewed SDK archive", - "external-gateway-health SDK install must retain: env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN", - 'external-gateway-health SDK install must retain: npm install --no-save --package-lock=false --ignore-scripts "${archives[0]}"', - "external-gateway-health must run only the credential-free external health test", - ]), - ); - }); - type FakeGateway = EventEmitter & { exitCode: number | null; signalCode: NodeJS.Signals | null; diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index ba110bea7d9..6def5e7c057 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -213,7 +213,6 @@ describe("packaged Blueprint Runner npm cache", () => { }); describe("packaged Blueprint Runner external target", () => { - // source-shape-contract: compatibility -- The installed command must select the protected root wrapper before its runtime behavior is trusted it( "executes the installed package command and fails before effects when the SDK is absent (#9872)", { timeout: 240_000 }, @@ -251,12 +250,6 @@ describe("packaged Blueprint Runner external target", () => { { encoding: "utf8", env: npmEnvironment() }, ); assertCommandSucceeded(extract, "root package archive extraction"); - const packedManifest = JSON.parse( - fs.readFileSync(path.join(installRoot, "package.json"), "utf8"), - ) as { bin?: Record }; - expect(packedManifest.bin?.["nemoclaw-blueprint-runner"]).toBe( - "./dist/lib/blueprint-runner.js", - ); fs.copyFileSync( path.join(REPOSITORY_ROOT, "package-lock.json"), path.join(installRoot, "package-lock.json"), From b70d24275302a67d37d61219b09ff022eff0d2c3 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 13:56:09 -0700 Subject: [PATCH 10/28] docs(blueprint): document external gateway status Signed-off-by: Apurv Kumaria --- docs/about/how-it-works.mdx | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 7146875e2ff..7cfe30f2682 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -115,6 +115,54 @@ NemoClaw separates host orchestration, agent-specific behavior, and sandbox defi This separation keeps host orchestration, agent-specific assets, and the sandbox definition at explicit lifecycle boundaries. +## Inspect an External OpenShell Gateway + +NemoClaw packages the experimental `nemoclaw-blueprint-runner` command for infrastructure that already owns an external OpenShell gateway. +This path supports target planning and one credential-free public health request for OpenShell `0.0.106`. +It does not manage the gateway or establish support for another OpenShell release. + +Create a blueprint directory with a `blueprint.yaml` file that contains one external target. +The target must use a bare HTTPS origin, the exact OpenShell release, an absolute CA bundle path, and an absolute authentication file path. + +```yaml +version: 1.0.0 +min_openshell_version: 0.0.106 +max_openshell_version: 0.0.106 +openshell_target: + endpoint: https://openshell.example.test:8443 + workspace: default + expected_release: 0.0.106 + lifecycle: external + trust: + ca_file: /var/run/openshell-target/ca.pem + authentication: + credential_file: /var/run/openshell-target/authentication +``` + +Run the target-only plan before the health request. +The plan validates local input, fingerprints the CA bundle, and prints a sanitized JSON plan without connecting. + +```bash +NEMOCLAW_BLUEPRINT_PATH=/absolute/path/to/blueprint nemoclaw-blueprint-runner plan +``` + +Run the external status command to make one public health request through the official OpenShell SDK. + +```bash +NEMOCLAW_BLUEPRINT_PATH=/absolute/path/to/blueprint nemoclaw-blueprint-runner status --external-target +``` + +A successful status result reports `healthy`, release `0.0.106`, and `compatible` in JSON. +The command validates authentication file metadata but does not read its contents or authenticate. +The command supplies the validated CA bundle to the SDK and uses normal TLS hostname verification. + + + This experimental path does not create, update, delete, or list gateways, workspaces, sandboxes, credentials, or policies. + It does not verify workspace readiness, machine authentication, or Kubernetes. + The infrastructure administrator must control the target hostname and its DNS resolution. + A failed status request makes no remote change, so correct the endpoint, CA bundle, file metadata, gateway health, or release mismatch before you retry. + + ## Readiness and Sandbox Creation Run `$$nemoclaw host probe` when you need a read-only system readiness report before onboarding. The report combines host and gateway observations, capabilities, qualifications, findings, evidence, and CLI provenance without changing system state. Onboarding consumes the same stable host and gateway entities and applies its explicit admission policy. It revalidates live facts after permitted preparation and when a saved onboarding session resumes. From 53ab70b9d5b0a7522640602235842447eb8d1106 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 14:13:21 -0700 Subject: [PATCH 11/28] docs(blueprint): clarify external gateway safety Signed-off-by: Apurv Kumaria --- docs/about/how-it-works.mdx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 7cfe30f2682..83d44637240 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -123,6 +123,10 @@ It does not manage the gateway or establish support for another OpenShell releas Create a blueprint directory with a `blueprint.yaml` file that contains one external target. The target must use a bare HTTPS origin, the exact OpenShell release, an absolute CA bundle path, and an absolute authentication file path. +The CA bundle and authentication file must be nonempty regular files, must not be symbolic links, and must be no larger than 1 MiB. +The CA bundle must contain only PEM CA certificates. +Both operations read the CA bundle, inspect only authentication file metadata, and never read the authentication file contents. +They do not create or remove either administrator-owned file. ```yaml version: 1.0.0 @@ -146,6 +150,12 @@ The plan validates local input, fingerprints the CA bundle, and prints a sanitiz NEMOCLAW_BLUEPRINT_PATH=/absolute/path/to/blueprint nemoclaw-blueprint-runner plan ``` + + The status command sends external traffic to the configured gateway. + The OpenShell SDK uses platform DNS without IP address pinning and verifies the server certificate and hostname against the supplied CA bundle. + You must control the target hostname and its DNS resolution. + + Run the external status command to make one public health request through the official OpenShell SDK. ```bash @@ -158,9 +168,9 @@ The command supplies the validated CA bundle to the SDK and uses normal TLS host This experimental path does not create, update, delete, or list gateways, workspaces, sandboxes, credentials, or policies. - It does not verify workspace readiness, machine authentication, or Kubernetes. - The infrastructure administrator must control the target hostname and its DNS resolution. + It does not establish workspace readiness or support machine authentication or Kubernetes. A failed status request makes no remote change, so correct the endpoint, CA bundle, file metadata, gateway health, or release mismatch before you retry. + If status reports that the approved OpenShell SDK is unavailable, restore a NemoClaw installation that includes `@nvidia/openshell-sdk` `0.0.106` before retrying. ## Readiness and Sandbox Creation From 0af779b610b950facbc971639f89b56fb8bb2dc6 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 14:22:46 -0700 Subject: [PATCH 12/28] test(e2e): cover packaged blueprint runner Signed-off-by: Apurv Kumaria --- test/e2e/support/cli-artifact-packaging.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/e2e/support/cli-artifact-packaging.test.ts b/test/e2e/support/cli-artifact-packaging.test.ts index 6483c4632d3..532a42dea06 100644 --- a/test/e2e/support/cli-artifact-packaging.test.ts +++ b/test/e2e/support/cli-artifact-packaging.test.ts @@ -68,15 +68,25 @@ exec ${JSON.stringify(systemTar)} "\${args[@]}" }).trim(); const dist = path.join(workspace, "dist"); + const packagedRunner = path.join(dist, "nemoclaw"); const shared = path.join(workspace, "nemoclaw", "dist", "shared"); - fs.mkdirSync(dist); + fs.mkdirSync(path.join(dist, "lib"), { recursive: true }); + fs.mkdirSync(path.join(packagedRunner, "blueprint"), { recursive: true }); fs.mkdirSync(shared, { recursive: true }); fs.writeFileSync(path.join(dist, "nemoclaw.js"), 'console.log("fixture");\n'); + fs.writeFileSync(path.join(dist, "lib", "blueprint-runner.js"), 'console.log("fixture");\n'); + fs.writeFileSync(path.join(packagedRunner, "package.json"), '{"type":"module"}\n'); + fs.writeFileSync( + path.join(packagedRunner, "blueprint", "runner.js"), + 'console.log("fixture");\n', + ); fs.writeFileSync( path.join(dist, "build-identity.json"), `${JSON.stringify({ nemoclawVersion: "0.0.0", sourceRevision: candidateSha })}\n`, ); for (const boundary of [ + "openshell-gateway-health-sdk.js", + "openshell-observation-boundary.cjs", "openshell-policy-boundary.cjs", "sandbox-name.cjs", "snapshot-sanitizer-boundary.cjs", From 051333d64a758cd7824835c6736bf93bff443335 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 15:30:01 -0700 Subject: [PATCH 13/28] test(blueprint): remove redundant deadline assertion Signed-off-by: Apurv Kumaria --- .../src/shared/openshell-gateway-health-sdk.test.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts b/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts index 2313ba2be7f..53c914a354a 100644 --- a/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts +++ b/nemoclaw/src/shared/openshell-gateway-health-sdk.test.ts @@ -82,18 +82,6 @@ describe("OpenShell SDK gateway health observer", () => { expect(Object.isFrozen(result.ok ? result.value : {})).toBe(true); }); - it("starts the deadline before loading the SDK (#9872)", async () => { - loadSdk.mockImplementation(async () => { - expect(timeoutSignal).toHaveBeenCalledOnce(); - return { connect, serviceStatus: SERVICE_STATUS }; - }); - const observer = createOpenShellSdkGatewayHealthObserver({ loadSdk, timeoutSignal }); - - await observer.observeHealth(request()); - - expect(connect).toHaveBeenCalledOnce(); - }); - it("loads the approved SDK exports through the default adapter (#9872)", async () => { const observer = createOpenShellSdkGatewayHealthObserver({ timeoutSignal }); From ce7bff89a3566858dd8ebb7c35d57139e48c0d96 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 16:02:36 -0700 Subject: [PATCH 14/28] test(blueprint): require missing health credential Signed-off-by: Apurv Kumaria --- test/e2e/README.md | 7 ++++--- test/e2e/live/external-gateway-health-helpers.ts | 6 +++--- ...ernal-gateway-health-workflow-boundary.test.ts | 15 +++++++++++++++ .../external-gateway-health-workflow-boundary.mts | 12 +++++++++++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index bc14315dc00..cbb5db84ce7 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -277,9 +277,10 @@ npx tsx tools/e2e/credential-free-tests.mts OpenShell target work in issue #9872. The trusted workflow downloads and verifies the exact OpenShell SDK archive with package-read permission. The candidate job receives the archive but no package credential. It calls a local -OpenShell 0.0.106 gateway over HTTPS with an explicit CA. The target validates -the authentication file metadata. It does not read the file contents or make an -authenticated gateway call. +OpenShell 0.0.106 gateway over HTTPS with an explicit CA. The target confirms +that its configured authentication file path does not exist before and after +the health request. A successful request proves that public health does not +require a credential read or make an authenticated gateway call. Use `jobs=external-gateway-health` for the manual pull request E2E run. The target records the expected release, reported release, public health status, diff --git a/test/e2e/live/external-gateway-health-helpers.ts b/test/e2e/live/external-gateway-health-helpers.ts index 04d0ca9adb6..178f9907f3c 100644 --- a/test/e2e/live/external-gateway-health-helpers.ts +++ b/test/e2e/live/external-gateway-health-helpers.ts @@ -27,7 +27,6 @@ import { } from "../fixtures/shell-probe.ts"; export const EXTERNAL_GATEWAY_HEALTH_TIMEOUT_MS = 3 * 60_000; -const UNUSED_AUTHENTICATION_SENTINEL = "unused-public-health-credential"; const BLUEPRINT_RUNNER = path.join( import.meta.dirname, "..", @@ -173,7 +172,7 @@ async function runBlueprintRunnerHealth( artifactName: "external-gateway-blueprint-runner-health", captureLimitBytes: 64 * 1024, env: { NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, - redactionValues: [privateStateRoot, UNUSED_AUTHENTICATION_SENTINEL], + redactionValues: [privateStateRoot], timeoutMs: 10_000, }, ); @@ -284,7 +283,7 @@ export async function runExternalGatewayHealthScenario({ const blueprintRoot = path.join(stateDir, "blueprint"); const authenticationPath = path.join(stateDir, "authentication"); fs.mkdirSync(blueprintRoot); - fs.writeFileSync(authenticationPath, `${UNUSED_AUTHENTICATION_SENTINEL}\n`, { mode: 0o600 }); + expect(fs.existsSync(authenticationPath)).toBe(false); fs.writeFileSync( path.join(blueprintRoot, "blueprint.yaml"), YAML.stringify({ @@ -311,6 +310,7 @@ export async function runExternalGatewayHealthScenario({ port, }); const status = await runBlueprintRunnerHealth(shellProbe, blueprintRoot, stateDir); + expect(fs.existsSync(authenticationPath)).toBe(false); expect(status.compatibility).toBe("compatible"); expect(status.gateway).toEqual({ release: OPENSHELL_V0106_QUALIFICATION.version, diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index c8435b13885..ccf7acc14da 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -6,6 +6,10 @@ import type { ChildProcess } from "node:child_process"; import { describe, expect, it, vi } from "vitest"; +import { + readExternalGatewayHealthHelper, + validateExternalGatewayHealthHelper, +} from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; import { stopExternalGatewayHealthGateway } from "../fixtures/external-gateway-health-process.ts"; describe("external gateway health workflow boundary", () => { @@ -73,4 +77,15 @@ describe("external gateway health workflow boundary", () => { vi.useRealTimers(); } }); + + it("rejects a readable credential fixture for public gateway health (#9872)", () => { + const source = readExternalGatewayHealthHelper(); + const readableCredentialFixture = source.replace( + "expect(fs.existsSync(authenticationPath)).toBe(false);", + 'fs.writeFileSync(authenticationPath, "readable fixture", { mode: 0o600 });', + ); + expect(validateExternalGatewayHealthHelper(readableCredentialFixture)).toContain( + "external gateway health helper must keep its credential path nonexistent", + ); + }); }); diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts index 72d038cfa47..67e32f0e998 100644 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ b/tools/e2e/external-gateway-health-workflow-boundary.mts @@ -308,7 +308,7 @@ export function validateExternalGatewayHealthHelper(source: string): string[] { '"status", "--external-target"', "captureLimitBytes: 64 * 1024", "NEMOCLAW_BLUEPRINT_PATH: blueprintRoot", - "redactionValues: [privateStateRoot, UNUSED_AUTHENTICATION_SENTINEL]", + "redactionValues: [privateStateRoot]", ]; if (required.some((fragment) => !source.includes(fragment))) { errors.push("external gateway health helper must run exact Blueprint Runner external status"); @@ -334,6 +334,16 @@ export function validateExternalGatewayHealthHelper(source: string): string[] { if (!source.includes("artifacts.addRedactionValues([stateDir]);")) { errors.push("external gateway health helper must redact its private fixture path"); } + const missingCredentialAssertions = + source.match(/expect\(fs\.existsSync\(authenticationPath\)\)\.toBe\(false\);/g)?.length ?? 0; + if ( + !source.includes('const authenticationPath = path.join(stateDir, "authentication");') || + !source.includes("authentication: { credential_file: authenticationPath }") || + missingCredentialAssertions < 2 || + source.includes("fs.writeFileSync(authenticationPath") + ) { + errors.push("external gateway health helper must keep its credential path nonexistent"); + } if (source.includes("@nvidia/openshell-sdk")) { errors.push( "external gateway health helper must not bypass the Runner with a direct SDK import", From 1796ea6303a59cdb9efbce031224ab78e6b786ba Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 16:22:49 -0700 Subject: [PATCH 15/28] fix(blueprint): avoid credentials during public health Signed-off-by: Apurv Kumaria --- docs/about/how-it-works.mdx | 10 ++++++---- .../openshell-external-target-boundary.cts | 19 +++++++++++-------- ...openshell-external-target-boundary.test.ts | 5 ++++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 83d44637240..1875891e4f0 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -123,10 +123,12 @@ It does not manage the gateway or establish support for another OpenShell releas Create a blueprint directory with a `blueprint.yaml` file that contains one external target. The target must use a bare HTTPS origin, the exact OpenShell release, an absolute CA bundle path, and an absolute authentication file path. -The CA bundle and authentication file must be nonempty regular files, must not be symbolic links, and must be no larger than 1 MiB. +The CA bundle must be a nonempty regular file, must not be a symbolic link, and must be no larger than 1 MiB. The CA bundle must contain only PEM CA certificates. -Both operations read the CA bundle, inspect only authentication file metadata, and never read the authentication file contents. -They do not create or remove either administrator-owned file. +Both operations read the CA bundle. +The plan requires a nonempty regular authentication file no larger than 1 MiB and inspects only its metadata. +The status command does not access the authentication file. +Neither operation creates or removes either administrator-owned file. ```yaml version: 1.0.0 @@ -163,7 +165,7 @@ NEMOCLAW_BLUEPRINT_PATH=/absolute/path/to/blueprint nemoclaw-blueprint-runner st ``` A successful status result reports `healthy`, release `0.0.106`, and `compatible` in JSON. -The command validates authentication file metadata but does not read its contents or authenticate. +The command does not access the authentication file or authenticate. The command supplies the validated CA bundle to the SDK and uses normal TLS hostname verification. diff --git a/nemoclaw/src/shared/openshell-external-target-boundary.cts b/nemoclaw/src/shared/openshell-external-target-boundary.cts index 59419dfca4a..e42f00a471c 100644 --- a/nemoclaw/src/shared/openshell-external-target-boundary.cts +++ b/nemoclaw/src/shared/openshell-external-target-boundary.cts @@ -322,6 +322,7 @@ function validateCaBundle(contents: Buffer): readonly Buffer[] { function validateExternalOpenShellTarget( value: unknown, compatibility: OpenShellCompatibilityRange, + validateAuthenticationFile: boolean, ): ValidatedExternalOpenShellTarget { const target = parseTarget(value); assertCompatibleRelease(target.expected_release, compatibility); @@ -329,11 +330,13 @@ function validateExternalOpenShellTarget( const caContents = readBoundedFile(target.trust.ca_file, "CA file", MAX_TRUST_FILE_BYTES); const caCertificates = validateCaBundle(caContents); - validateFileReference( - target.authentication.credential_file, - "authentication file", - MAX_AUTHENTICATION_FILE_BYTES, - ); + if (validateAuthenticationFile) { + validateFileReference( + target.authentication.credential_file, + "authentication file", + MAX_AUTHENTICATION_FILE_BYTES, + ); + } const caFingerprint = createHash("sha256"); for (const certificate of caCertificates) { @@ -356,18 +359,18 @@ export function buildSanitizedExternalOpenShellTargetPlan( value: unknown, compatibility: OpenShellCompatibilityRange, ): SanitizedExternalOpenShellTargetPlan { - return validateExternalOpenShellTarget(value, compatibility).plan; + return validateExternalOpenShellTarget(value, compatibility, true).plan; } /** * Give one caller the validated CA bytes without exposing either input path. - * Authentication contents remain unread because this boundary inspects only their file metadata. + * The public health path does not access the authentication file. */ export async function withExternalOpenShellTargetCa( value: unknown, compatibility: OpenShellCompatibilityRange, useTarget: (plan: SanitizedExternalOpenShellTargetPlan, caContents: Buffer) => Promise, ): Promise { - const validated = validateExternalOpenShellTarget(value, compatibility); + const validated = validateExternalOpenShellTarget(value, compatibility, false); return useTarget(validated.plan, Buffer.from(validated.caContents)); } diff --git a/nemoclaw/src/shared/openshell-external-target-boundary.test.ts b/nemoclaw/src/shared/openshell-external-target-boundary.test.ts index 0c1d63f4240..fdc39e5684f 100644 --- a/nemoclaw/src/shared/openshell-external-target-boundary.test.ts +++ b/nemoclaw/src/shared/openshell-external-target-boundary.test.ts @@ -155,7 +155,9 @@ describe("external OpenShell target boundary", () => { expect(Object.isFrozen(plan)).toBe(true); }); - it("passes validated CA bytes without reading authentication contents (#9872)", async () => { + it("passes public-health CA bytes when the authentication file is absent (#9872)", async () => { + fileContents.delete(AUTHENTICATION_FILE); + const result = await withExternalOpenShellTargetCa( externalTarget(), COMPATIBILITY, @@ -166,6 +168,7 @@ describe("external OpenShell target boundary", () => { expect(result.caContents.toString("utf8")).toBe(CA_PEM); expect(readFilePaths).toContain(CA_FILE); expect(readFilePaths).not.toContain(AUTHENTICATION_FILE); + expect(fsMocks.openSync).not.toHaveBeenCalledWith(AUTHENTICATION_FILE, expect.any(Number)); }); it.each([ From 61a79060568b81beb999e2c11f577d4a917a0386 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 16:43:46 -0700 Subject: [PATCH 16/28] fix(blueprint): align credential-free status contract Signed-off-by: Apurv Kumaria --- schemas/blueprint.schema.json | 6 ++-- ...l-gateway-health-workflow-boundary.test.ts | 15 ---------- test/onboarding/blueprint-name-schema.test.ts | 29 +++++++++++++++++++ ...ernal-gateway-health-workflow-boundary.mts | 10 ------- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/schemas/blueprint.schema.json b/schemas/blueprint.schema.json index f4df45028c9..7b3341a173a 100644 --- a/schemas/blueprint.schema.json +++ b/schemas/blueprint.schema.json @@ -172,7 +172,7 @@ "$defs": { "externalOpenShellTarget": { "type": "object", - "description": "Explicit externally managed OpenShell gateway target for target-only planning and credential-free public health status. File references use POSIX-rooted absolute paths. Planning validates and sanitizes target inputs without connecting. Status checks authentication file metadata, then uses the explicit HTTPS endpoint and validated CA bundle for one public health request. Status does not read authentication file contents or authenticate. This contract does not establish remote readiness or support for machine authentication or Kubernetes.", + "description": "Explicit externally managed OpenShell gateway target for target-only planning and credential-free public health status. File references use POSIX-rooted absolute paths. Planning validates and sanitizes target inputs without connecting. Status uses the explicit HTTPS endpoint and validated CA bundle for one public health request. Status does not access the authentication file or authenticate. This contract does not establish remote readiness or support for machine authentication or Kubernetes.", "required": [ "endpoint", "workspace", @@ -220,13 +220,13 @@ "type": "object", "required": ["credential_file"], "additionalProperties": false, - "description": "File-backed authentication reference required for target-only planning and status. Both operations check file metadata only. Neither reads its contents, authenticates, or selects or validates a machine-authentication protocol.", + "description": "File-backed authentication reference required in the target shape for target-only planning and status. Planning validates the referenced file metadata without reading its contents. Status accepts the reference but does not access or validate the referenced file. Neither operation authenticates or selects or validates a machine-authentication protocol.", "properties": { "credential_file": { "type": "string", "minLength": 1, "pattern": "^/", - "description": "POSIX-rooted absolute path to a non-empty regular authentication file no larger than 1 MiB. Symbolic-link paths are not accepted. The planning and public health status operations inspect only file metadata. Neither operation reads the file contents or returns the path." + "description": "POSIX-rooted absolute authentication file path. Planning requires a non-empty regular file no larger than 1 MiB, rejects symbolic links, and inspects only file metadata. Public health status does not access, validate, or require the referenced file. Neither operation returns the path." } } } diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index ccf7acc14da..c8435b13885 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -6,10 +6,6 @@ import type { ChildProcess } from "node:child_process"; import { describe, expect, it, vi } from "vitest"; -import { - readExternalGatewayHealthHelper, - validateExternalGatewayHealthHelper, -} from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; import { stopExternalGatewayHealthGateway } from "../fixtures/external-gateway-health-process.ts"; describe("external gateway health workflow boundary", () => { @@ -77,15 +73,4 @@ describe("external gateway health workflow boundary", () => { vi.useRealTimers(); } }); - - it("rejects a readable credential fixture for public gateway health (#9872)", () => { - const source = readExternalGatewayHealthHelper(); - const readableCredentialFixture = source.replace( - "expect(fs.existsSync(authenticationPath)).toBe(false);", - 'fs.writeFileSync(authenticationPath, "readable fixture", { mode: 0o600 });', - ); - expect(validateExternalGatewayHealthHelper(readableCredentialFixture)).toContain( - "external gateway health helper must keep its credential path nonexistent", - ); - }); }); diff --git a/test/onboarding/blueprint-name-schema.test.ts b/test/onboarding/blueprint-name-schema.test.ts index a71a10c4a76..385a638aeb6 100644 --- a/test/onboarding/blueprint-name-schema.test.ts +++ b/test/onboarding/blueprint-name-schema.test.ts @@ -1,11 +1,26 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { readFileSync } from "node:fs"; + import { describe, expect, it } from "vitest"; import { compileConfigSchema } from "../../scripts/validate-configs.mts"; const validate = compileConfigSchema("schemas/blueprint.schema.json"); +const blueprintSchema = JSON.parse(readFileSync("schemas/blueprint.schema.json", "utf8")) as { + $defs: { + externalOpenShellTarget: { + description: string; + properties: { + authentication: { + description: string; + properties: { credential_file: { description: string } }; + }; + }; + }; + }; +}; interface BlueprintFixture { version: string; @@ -96,6 +111,20 @@ describe("blueprint name schema", () => { }); describe("blueprint external OpenShell target schema", () => { + it("documents credential validation for plan but not public health status (#9872)", () => { + const target = blueprintSchema.$defs.externalOpenShellTarget; + const descriptions = [ + target.description, + target.properties.authentication.description, + target.properties.authentication.properties.credential_file.description, + ]; + + expect(descriptions[0]).toContain("Status does not access the authentication file"); + expect(descriptions[1]).toContain("Planning validates the referenced file metadata"); + expect(descriptions[1]).toContain("Status accepts the reference but does not access"); + expect(descriptions[2]).toContain("Public health status does not access, validate, or require"); + }); + it("accepts one explicit target without managed components", () => { expect(validate(blueprintWithExternalTarget()), JSON.stringify(validate.errors)).toBe(true); }); diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts index 67e32f0e998..6f4a2dc4c1f 100644 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ b/tools/e2e/external-gateway-health-workflow-boundary.mts @@ -334,16 +334,6 @@ export function validateExternalGatewayHealthHelper(source: string): string[] { if (!source.includes("artifacts.addRedactionValues([stateDir]);")) { errors.push("external gateway health helper must redact its private fixture path"); } - const missingCredentialAssertions = - source.match(/expect\(fs\.existsSync\(authenticationPath\)\)\.toBe\(false\);/g)?.length ?? 0; - if ( - !source.includes('const authenticationPath = path.join(stateDir, "authentication");') || - !source.includes("authentication: { credential_file: authenticationPath }") || - missingCredentialAssertions < 2 || - source.includes("fs.writeFileSync(authenticationPath") - ) { - errors.push("external gateway health helper must keep its credential path nonexistent"); - } if (source.includes("@nvidia/openshell-sdk")) { errors.push( "external gateway health helper must not bypass the Runner with a direct SDK import", From 24f96d4239725759ba2431806ff77798d115e68c Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 17:01:33 -0700 Subject: [PATCH 17/28] test(schema): remove wording-only assertion Signed-off-by: Apurv Kumaria --- test/onboarding/blueprint-name-schema.test.ts | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/test/onboarding/blueprint-name-schema.test.ts b/test/onboarding/blueprint-name-schema.test.ts index 385a638aeb6..a71a10c4a76 100644 --- a/test/onboarding/blueprint-name-schema.test.ts +++ b/test/onboarding/blueprint-name-schema.test.ts @@ -1,26 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { readFileSync } from "node:fs"; - import { describe, expect, it } from "vitest"; import { compileConfigSchema } from "../../scripts/validate-configs.mts"; const validate = compileConfigSchema("schemas/blueprint.schema.json"); -const blueprintSchema = JSON.parse(readFileSync("schemas/blueprint.schema.json", "utf8")) as { - $defs: { - externalOpenShellTarget: { - description: string; - properties: { - authentication: { - description: string; - properties: { credential_file: { description: string } }; - }; - }; - }; - }; -}; interface BlueprintFixture { version: string; @@ -111,20 +96,6 @@ describe("blueprint name schema", () => { }); describe("blueprint external OpenShell target schema", () => { - it("documents credential validation for plan but not public health status (#9872)", () => { - const target = blueprintSchema.$defs.externalOpenShellTarget; - const descriptions = [ - target.description, - target.properties.authentication.description, - target.properties.authentication.properties.credential_file.description, - ]; - - expect(descriptions[0]).toContain("Status does not access the authentication file"); - expect(descriptions[1]).toContain("Planning validates the referenced file metadata"); - expect(descriptions[1]).toContain("Status accepts the reference but does not access"); - expect(descriptions[2]).toContain("Public health status does not access, validate, or require"); - }); - it("accepts one explicit target without managed components", () => { expect(validate(blueprintWithExternalTarget()), JSON.stringify(validate.errors)).toBe(true); }); From 2dec43b61e6deebab72db04a411ddd56855badf3 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 17:09:14 -0700 Subject: [PATCH 18/28] docs(openshell): clarify status recovery Signed-off-by: Apurv Kumaria --- docs/about/how-it-works.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 1875891e4f0..248bfbf86c0 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -171,7 +171,7 @@ The command supplies the validated CA bundle to the SDK and uses normal TLS host This experimental path does not create, update, delete, or list gateways, workspaces, sandboxes, credentials, or policies. It does not establish workspace readiness or support machine authentication or Kubernetes. - A failed status request makes no remote change, so correct the endpoint, CA bundle, file metadata, gateway health, or release mismatch before you retry. + A failed status request makes no remote change, so correct the endpoint, CA bundle or its file metadata, gateway health, or release mismatch before you retry. If status reports that the approved OpenShell SDK is unavailable, restore a NemoClaw installation that includes `@nvidia/openshell-sdk` `0.0.106` before retrying. From abed18e4bd5e7aa7098b4fd56832c4d5753142e1 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 19:06:51 -0700 Subject: [PATCH 19/28] docs(openshell): clarify public health path Signed-off-by: Apurv Kumaria --- docs/about/how-it-works.mdx | 8 +++++--- .../blueprint-external-target-timeout.test.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 248bfbf86c0..c37a9ba2022 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -145,8 +145,8 @@ openshell_target: credential_file: /var/run/openshell-target/authentication ``` -Run the target-only plan before the health request. -The plan validates local input, fingerprints the CA bundle, and prints a sanitized JSON plan without connecting. +Run the target-only plan when you need to validate the complete target configuration and print a sanitized plan. +Planning requires the referenced authentication file, validates local input, fingerprints the CA bundle, and does not connect. ```bash NEMOCLAW_BLUEPRINT_PATH=/absolute/path/to/blueprint nemoclaw-blueprint-runner plan @@ -158,7 +158,9 @@ NEMOCLAW_BLUEPRINT_PATH=/absolute/path/to/blueprint nemoclaw-blueprint-runner pl You must control the target hostname and its DNS resolution. -Run the external status command to make one public health request through the official OpenShell SDK. +Run the external status command directly when you only need public health. +Status requires the absolute authentication file reference but does not access the file. +It makes one public health request through the official OpenShell SDK. ```bash NEMOCLAW_BLUEPRINT_PATH=/absolute/path/to/blueprint nemoclaw-blueprint-runner status --external-target diff --git a/test/package-contract/blueprint-external-target-timeout.test.ts b/test/package-contract/blueprint-external-target-timeout.test.ts index ab2dde56146..3277ae92f94 100644 --- a/test/package-contract/blueprint-external-target-timeout.test.ts +++ b/test/package-contract/blueprint-external-target-timeout.test.ts @@ -239,7 +239,7 @@ function expectFixedReachabilityFailure( describe("packaged Blueprint Runner external health deadline", () => { it( - "cancels official SDK health and exits when the TLS peer stalls (#9872)", + "returns a fixed reachability failure within the deadline when the TLS peer stalls (#9872)", { timeout: 20_000 }, async () => { const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-health-timeout-")); From 5897d169a256913d08f72304b313d0f1ee765dc2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 31 Aug 2026 19:16:09 -0700 Subject: [PATCH 20/28] fix(openshell): address advisor findings Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 122 +------------- ...ypescript-sdk-0.0.106-dependency-review.md | 15 +- scripts/e2e/package-cli-artifact.sh | 153 ++++++++++++++++++ .../support/cli-artifact-packaging.test.ts | 10 +- .../cli-artifact-workflow-boundary.test.ts | 22 +-- .../blueprint-external-target-plan.test.ts | 114 ------------- .../blueprint-runner-unsafe-diagnostic.ts | 8 - tools/e2e/cli-artifact-workflow-boundary.mts | 41 +---- 8 files changed, 174 insertions(+), 311 deletions(-) create mode 100755 scripts/e2e/package-cli-artifact.sh delete mode 100644 test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3fc577a3b7f..f26392d7998 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -748,127 +748,7 @@ jobs: RUN_ID: ${{ github.run_id }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash - run: | - set -euo pipefail - [[ "$CANDIDATE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || - { echo "::error::candidate repository is invalid"; exit 1; } - [[ "$CANDIDATE_SHA" =~ ^[a-f0-9]{40}$ ]] || - { echo "::error::candidate SHA must be a lowercase 40-character SHA"; exit 1; } - [[ "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] || - { echo "::error::workflow SHA must be a lowercase 40-character SHA"; exit 1; } - [[ "$(git rev-parse --verify HEAD)" == "$CANDIDATE_SHA" ]] || - { echo "::error::checked-out commit does not match the artifact candidate SHA"; exit 1; } - [[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ && "$RUN_ID" =~ ^[1-9][0-9]*$ ]] || - { echo "::error::workflow run identity is invalid"; exit 1; } - for required_file in \ - dist/nemoclaw.js \ - dist/build-identity.json \ - dist/lib/blueprint-runner.js \ - dist/nemoclaw/package.json \ - dist/nemoclaw/blueprint/runner.js; do - [[ -f "$required_file" && ! -L "$required_file" && -s "$required_file" ]] || - { echo "::error::candidate CLI build output is missing or is not a nonempty regular file: $required_file"; exit 1; } - done - for boundary in \ - openshell-gateway-health-sdk.js \ - openshell-observation-boundary.cjs \ - openshell-policy-boundary.cjs \ - sandbox-name.cjs \ - snapshot-sanitizer-boundary.cjs; do - boundary_path="nemoclaw/dist/shared/$boundary" - [[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]] || - { echo "::error::candidate CLI build shared module is missing or is not a nonempty regular file: $boundary"; exit 1; } - done - - jq -e --arg candidateSha "$CANDIDATE_SHA" ' - type == "object" and - (keys | sort) == ["nemoclawVersion", "sourceRevision"] and - (.nemoclawVersion | strings | length > 0) and - .sourceRevision == $candidateSha - ' dist/build-identity.json >/dev/null || - { echo "::error::candidate CLI build identity does not match the candidate commit SHA"; exit 1; } - jq -e ' - type == "object" and - (keys | sort) == ["type"] and - .type == "module" - ' dist/nemoclaw/package.json >/dev/null || - { echo "::error::candidate Blueprint Runner module boundary is invalid"; exit 1; } - - artifact_catalog="dist/e2e-managed-image-catalog.json" - [[ ! -e "$artifact_catalog" && ! -L "$artifact_catalog" ]] || { - echo "::error::candidate build created the managed-image catalog path" >&2 - exit 1 - } - - artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact" - install -d -m 0700 "$artifact_dir" - payload="$artifact_dir/nemoclaw-cli.tar" - manifest="$artifact_dir/manifest.json" - tar \ - --sort=name \ - --mtime=@0 \ - --owner=0 \ - --group=0 \ - --numeric-owner \ - -cf "$payload" \ - dist \ - nemoclaw/dist/shared - payload_sha256="$(sha256sum "$payload" | awk '{print $1}')" - source_tree="$(git rev-parse 'HEAD^{tree}')" - lockfile_sha256="$(sha256sum package-lock.json | awk '{print $1}')" - node_version="$(node --version)" - npm_version="$(npm --version)" - artifact_name="nemoclaw-cli-${CANDIDATE_SHA}-${payload_sha256}" - - jq -n \ - --arg artifactName "$artifact_name" \ - --arg buildCommand "npm run build:cli" \ - --arg candidateRepository "$CANDIDATE_REPOSITORY" \ - --arg candidateSha "$CANDIDATE_SHA" \ - --arg lockfileSha256 "$lockfile_sha256" \ - --arg nodeVersion "$node_version" \ - --arg npmVersion "$npm_version" \ - --arg payloadFile "nemoclaw-cli.tar" \ - --arg payloadSha256 "$payload_sha256" \ - --arg runAttempt "$RUN_ATTEMPT" \ - --arg runId "$RUN_ID" \ - --arg runnerArch "$RUNNER_ARCH" \ - --arg runnerOs "$RUNNER_OS" \ - --arg sourceTree "$source_tree" \ - --arg workflowSha "$WORKFLOW_SHA" \ - '{ - kind: "nemoclaw-e2e-cli-artifact-v1", - artifactName: $artifactName, - candidate: { - repository: $candidateRepository, - sha: $candidateSha, - sourceTree: $sourceTree, - lockfileSha256: $lockfileSha256 - }, - workflow: { - sha: $workflowSha, - runId: $runId, - runAttempt: $runAttempt - }, - toolchain: { - node: $nodeVersion, - npm: $npmVersion, - runnerOs: $runnerOs, - runnerArch: $runnerArch - }, - build: { - command: $buildCommand, - sourceRevision: $candidateSha - }, - payload: { - file: $payloadFile, - sha256: $payloadSha256 - } - }' >"$manifest" - chmod 0600 "$manifest" "$payload" - printf 'artifact_name=%s\n' "$artifact_name" >>"$GITHUB_OUTPUT" - printf 'candidate_sha=%s\n' "$CANDIDATE_SHA" >>"$GITHUB_OUTPUT" - printf 'payload_sha256=%s\n' "$payload_sha256" >>"$GITHUB_OUTPUT" + run: scripts/e2e/package-cli-artifact.sh - id: upload_cli_artifact name: Publish content-addressed CLI artifact diff --git a/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md b/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md index 6b4062eb0aa..42ddc30e3b4 100644 --- a/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md +++ b/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md @@ -117,15 +117,18 @@ The image software bill of materials and third-party notice must bind to the ins - Severity and confidence: high, high confidence. - Failure mode: a validated hostname resolves or rebinds to a local service when the request starts. -- Control: the official SDK retains TLS certificate and hostname verification against the explicit - CA, and the slice sends only unauthenticated public health. The accepted initial contract treats - platform infrastructure administrators as trusted. +- Control: the accepted initial contract treats cluster and storage administrators as trusted + infrastructure actors. The configured endpoint and CA come from those administrators. The + operation sends only unauthenticated public health and retains TLS certificate and hostname + verification. - Residual: SDK 0.0.106 does not expose a pinned lookup. A hostname can resolve differently between validation and connection, including to a local address. TLS identity verification prevents an unrelated local service from impersonating the configured gateway, but it does not prevent the - connection attempt. -- Verification: wrong-CA and wrong-hostname package tests; independent security review must keep - this residual visible. + connection attempt. Restrict deployment authority to the trusted administrators named in issue + #9872. +- Verification: wrong-CA and wrong-hostname package tests protect TLS identity. The live test uses + an administrator-selected private gateway address. Independent security review must keep this + accepted trust boundary visible. ### DEP-5: Transport outlives the bounded observation diff --git a/scripts/e2e/package-cli-artifact.sh b/scripts/e2e/package-cli-artifact.sh new file mode 100755 index 00000000000..287e08ff992 --- /dev/null +++ b/scripts/e2e/package-cli-artifact.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +[[ "$CANDIDATE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] \ + || { + echo "::error::candidate repository is invalid" + exit 1 + } +[[ "$CANDIDATE_SHA" =~ ^[a-f0-9]{40}$ ]] \ + || { + echo "::error::candidate SHA must be a lowercase 40-character SHA" + exit 1 + } +[[ "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] \ + || { + echo "::error::workflow SHA must be a lowercase 40-character SHA" + exit 1 + } +[[ "$(git rev-parse --verify HEAD)" == "$CANDIDATE_SHA" ]] \ + || { + echo "::error::checked-out commit does not match the artifact candidate SHA" + exit 1 + } +[[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ && "$RUN_ID" =~ ^[1-9][0-9]*$ ]] \ + || { + echo "::error::workflow run identity is invalid" + exit 1 + } +for required_file in \ + dist/nemoclaw.js \ + dist/build-identity.json \ + dist/lib/blueprint-runner.js \ + dist/nemoclaw/package.json \ + dist/nemoclaw/blueprint/runner.js; do + [[ -f "$required_file" && ! -L "$required_file" && -s "$required_file" ]] \ + || { + echo "::error::candidate CLI build output is missing or is not a nonempty regular file: $required_file" + exit 1 + } +done +for boundary in \ + openshell-gateway-health-sdk.js \ + openshell-observation-boundary.cjs \ + openshell-policy-boundary.cjs \ + sandbox-name.cjs \ + snapshot-sanitizer-boundary.cjs; do + boundary_path="nemoclaw/dist/shared/$boundary" + [[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]] \ + || { + echo "::error::candidate CLI build shared module is missing or is not a nonempty regular file: $boundary" + exit 1 + } +done + +jq -e --arg candidateSha "$CANDIDATE_SHA" ' + type == "object" and + (keys | sort) == ["nemoclawVersion", "sourceRevision"] and + (.nemoclawVersion | strings | length > 0) and + .sourceRevision == $candidateSha +' dist/build-identity.json >/dev/null \ + || { + echo "::error::candidate CLI build identity does not match the candidate commit SHA" + exit 1 + } +jq -e ' + type == "object" and + (keys | sort) == ["type"] and + .type == "module" +' dist/nemoclaw/package.json >/dev/null \ + || { + echo "::error::candidate Blueprint Runner module boundary is invalid" + exit 1 + } + +artifact_catalog="dist/e2e-managed-image-catalog.json" +[[ ! -e "$artifact_catalog" && ! -L "$artifact_catalog" ]] || { + echo "::error::candidate build created the managed-image catalog path" >&2 + exit 1 +} + +artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact" +install -d -m 0700 "$artifact_dir" +payload="$artifact_dir/nemoclaw-cli.tar" +manifest="$artifact_dir/manifest.json" +tar \ + --sort=name \ + --mtime=@0 \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + -cf "$payload" \ + dist \ + nemoclaw/dist/shared +payload_sha256="$(sha256sum "$payload" | awk '{print $1}')" +source_tree="$(git rev-parse 'HEAD^{tree}')" +lockfile_sha256="$(sha256sum package-lock.json | awk '{print $1}')" +node_version="$(node --version)" +npm_version="$(npm --version)" +artifact_name="nemoclaw-cli-${CANDIDATE_SHA}-${payload_sha256}" + +jq -n \ + --arg artifactName "$artifact_name" \ + --arg buildCommand "npm run build:cli" \ + --arg candidateRepository "$CANDIDATE_REPOSITORY" \ + --arg candidateSha "$CANDIDATE_SHA" \ + --arg lockfileSha256 "$lockfile_sha256" \ + --arg nodeVersion "$node_version" \ + --arg npmVersion "$npm_version" \ + --arg payloadFile "nemoclaw-cli.tar" \ + --arg payloadSha256 "$payload_sha256" \ + --arg runAttempt "$RUN_ATTEMPT" \ + --arg runId "$RUN_ID" \ + --arg runnerArch "$RUNNER_ARCH" \ + --arg runnerOs "$RUNNER_OS" \ + --arg sourceTree "$source_tree" \ + --arg workflowSha "$WORKFLOW_SHA" \ + '{ + kind: "nemoclaw-e2e-cli-artifact-v1", + artifactName: $artifactName, + candidate: { + repository: $candidateRepository, + sha: $candidateSha, + sourceTree: $sourceTree, + lockfileSha256: $lockfileSha256 + }, + workflow: { + sha: $workflowSha, + runId: $runId, + runAttempt: $runAttempt + }, + toolchain: { + node: $nodeVersion, + npm: $npmVersion, + runnerOs: $runnerOs, + runnerArch: $runnerArch + }, + build: { + command: $buildCommand, + sourceRevision: $candidateSha + }, + payload: { + file: $payloadFile, + sha256: $payloadSha256 + } + }' >"$manifest" +chmod 0600 "$manifest" "$payload" +{ + printf 'artifact_name=%s\n' "$artifact_name" + printf 'candidate_sha=%s\n' "$CANDIDATE_SHA" + printf 'payload_sha256=%s\n' "$payload_sha256" +} >>"$GITHUB_OUTPUT" diff --git a/test/e2e/support/cli-artifact-packaging.test.ts b/test/e2e/support/cli-artifact-packaging.test.ts index 532a42dea06..5984efe34ab 100644 --- a/test/e2e/support/cli-artifact-packaging.test.ts +++ b/test/e2e/support/cli-artifact-packaging.test.ts @@ -8,8 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { CLI_ARTIFACT_PACKAGE_STEP } from "../../../tools/e2e/cli-artifact-workflow-boundary.mts"; -import { readWorkflow, type Workflow } from "../../helpers/e2e-workflow-contract"; +import { CLI_ARTIFACT_PACKAGE_SCRIPT } from "../../../tools/e2e/cli-artifact-workflow-boundary.mts"; type CatalogInput = "absent" | "file" | "symlink"; @@ -97,12 +96,7 @@ exec ${JSON.stringify(systemTar)} "\${args[@]}" const catalog = path.join(dist, "e2e-managed-image-catalog.json"); CATALOG_INPUT_WRITERS[catalogInput](catalog); - const workflow = readWorkflow() as Workflow; - const packageStep = workflow.jobs["generate-matrix"]?.steps?.find( - (step) => step.name === CLI_ARTIFACT_PACKAGE_STEP, - ); - expect(packageStep?.run).toEqual(expect.any(String)); - const result = spawnSync("bash", ["-c", packageStep!.run!], { + const result = spawnSync("bash", [path.resolve(CLI_ARTIFACT_PACKAGE_SCRIPT)], { cwd: workspace, encoding: "utf8", env: { diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index f74be2f97b0..fd7cc1044af 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -8,6 +8,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { + CLI_ARTIFACT_PACKAGE_SCRIPT, CLI_ARTIFACT_PACKAGE_STEP, CLI_ARTIFACT_PUBLISH_STEP, validateCliArtifactRestoreAction, @@ -25,8 +26,6 @@ const CANDIDATE_SHA = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8", }).trim(); const PAYLOAD_SHA256 = "b".repeat(64); -const CONTENT_ADDRESSED_ARTIFACT_NAME = `artifact_name="nemoclaw-cli-\${CANDIDATE_SHA}-\${payload_sha256}"`; -const UNBOUND_ARTIFACT_NAME = `artifact_name="nemoclaw-cli-\${CANDIDATE_SHA}"`; function runIdentityValidation(overrides: Record = {}, consumerAttempt = "1") { const action = readYaml(".github/actions/restore-e2e-cli-artifact/action.yaml"); @@ -909,17 +908,7 @@ describe("exact-commit CLI artifact workflow boundary", () => { "${{ steps.upload_cli_artifact.outputs.artifact-url }}"; const packageStep = requireStep(workflow, "generate-matrix", CLI_ARTIFACT_PACKAGE_STEP); packageStep.env!.WORKFLOW_SHA = "${{ inputs.checkout_sha }}"; - packageStep.run = packageStep.run!.replace( - CONTENT_ADDRESSED_ARTIFACT_NAME, - UNBOUND_ARTIFACT_NAME, - ); - packageStep.run = packageStep - .run!.replace("openshell-gateway-health-sdk.js", "missing-gateway-health-sdk.js") - .replace("openshell-observation-boundary.cjs", "missing-observation-boundary.cjs") - .replace("sandbox-name.cjs", "missing-boundary.cjs"); - packageStep.run = packageStep - .run!.replaceAll("dist/nemoclaw/package.json", "dist/nemoclaw/missing-package.json") - .replace('.type == "module"', '.type == "commonjs"'); + packageStep.run = `bash ${CLI_ARTIFACT_PACKAGE_SCRIPT}`; const uploadStep = requireStep(workflow, "generate-matrix", CLI_ARTIFACT_PUBLISH_STEP); uploadStep.uses = "actions/upload-artifact@v7"; @@ -927,12 +916,7 @@ describe("exact-commit CLI artifact workflow boundary", () => { expect.arrayContaining([ "generate-matrix must expose exact cli_artifact_provenance provenance", "CLI artifact package step must bind candidate and trusted workflow identities explicitly", - `CLI artifact package step must contain ${CONTENT_ADDRESSED_ARTIFACT_NAME}`, - "CLI artifact package step must contain openshell-gateway-health-sdk.js", - "CLI artifact package step must contain openshell-observation-boundary.cjs", - "CLI artifact package step must contain sandbox-name.cjs", - "CLI artifact package step must contain dist/nemoclaw/package.json", - 'CLI artifact package step must contain .type == "module"', + `CLI artifact package step must invoke ${CLI_ARTIFACT_PACKAGE_SCRIPT} directly`, "CLI artifact upload must use the immutable content-addressed upload contract", ]), ); diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 6def5e7c057..dd80176448f 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -13,16 +13,8 @@ import YAML from "yaml"; const REPOSITORY_ROOT = path.join(import.meta.dirname, "..", ".."); const OPENSHELL_SDK_PACKAGE = "@nvidia/openshell-sdk"; -const UNSAFE_RUNNER_FIXTURE = path.join( - REPOSITORY_ROOT, - "test", - "package-contract", - "fixtures", - "blueprint-runner-unsafe-diagnostic.ts", -); const PRIVATE_AUTHENTICATION_CONTENTS = "opaque-private-authentication-material"; const PRIVATE_AMBIENT_CONTENTS = "opaque-ambient-gateway-material"; -const PRIVATE_ENTRY_SECRET = `nvapi-${"A".repeat(64)}`; const CA_PEM = rootCertificates[0]!; const UNSAFE_DIAGNOSTIC_CHARACTERS = /[\u0000-\u001f\u007f-\u009f\u200b-\u200f\u2028-\u202e\u2060-\u206f\ufeff]/u; @@ -64,15 +56,6 @@ function packagePath(root: string, packageName: string): string { return path.join(root, "node_modules", ...packageName.split("/")); } -function installedPackageIdentity( - installRoot: string, - packageName: string, -): Readonly<{ name: unknown; version: unknown }> { - const manifestPath = path.join(packagePath(installRoot, packageName), "package.json"); - const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as Record; - return { name: manifest.name, version: manifest.version }; -} - function writeRuntimeProbe(probePath: string): void { fs.writeFileSync( probePath, @@ -289,88 +272,12 @@ describe("packaged Blueprint Runner external target", () => { ); const installedPackage = installRoot; - const installedRunner = path.join( - installedPackage, - "nemoclaw", - "dist", - "blueprint", - "runner.js", - ); - const installedRuntimeRunner = path.join( - installedPackage, - "dist", - "nemoclaw", - "blueprint", - "runner.js", - ); - const installedRootObserver = path.join( - installedPackage, - "dist", - "lib", - "adapters", - "openshell", - "sandbox-observer.js", - ); - const installedSharedSdkObserver = path.join( - installedPackage, - "nemoclaw", - "dist", - "shared", - "openshell-gateway-health-sdk.js", - ); - const installedRuntimeSdkObserver = path.join( - installedPackage, - "dist", - "nemoclaw", - "shared", - "openshell-gateway-health-sdk.js", - ); const installedBinary = path.join( consumerRoot, "node_modules", ".bin", "nemoclaw-blueprint-runner", ); - expect(fs.existsSync(installedRunner)).toBe(true); - expect(fs.existsSync(installedRootObserver)).toBe(true); - expect(fs.existsSync(installedSharedSdkObserver)).toBe(true); - expect(fs.existsSync(installedRuntimeSdkObserver)).toBe(false); - expect(fs.existsSync(path.join(installedPackage, "nemoclaw", "src"))).toBe(false); - expect(fs.existsSync(installedRuntimeRunner)).toBe(true); - expect( - JSON.parse( - fs.readFileSync( - path.join(installedPackage, "dist", "nemoclaw", "package.json"), - "utf8", - ), - ), - ).toEqual({ type: "module" }); - expect(installedPackageIdentity(installRoot, "@bufbuild/protobuf")).toEqual({ - name: "@bufbuild/protobuf", - version: "2.12.1", - }); - expect(installedPackageIdentity(installRoot, "@connectrpc/connect")).toEqual({ - name: "@connectrpc/connect", - version: "2.1.2", - }); - expect(installedPackageIdentity(installRoot, "@connectrpc/connect-node")).toEqual({ - name: "@connectrpc/connect-node", - version: "2.1.2", - }); - expect(installedPackageIdentity(installRoot, OPENSHELL_SDK_PACKAGE)).toEqual({ - name: OPENSHELL_SDK_PACKAGE, - version: "0.0.106", - }); - const installedSdkRoot = packagePath(installRoot, OPENSHELL_SDK_PACKAGE); - expect( - fs.existsSync(path.join(installedSdkRoot, "node_modules", "@bufbuild", "protobuf")), - ).toBe(false); - expect( - fs.existsSync(path.join(installedSdkRoot, "node_modules", "@connectrpc", "connect")), - ).toBe(false); - expect( - fs.existsSync(path.join(installedSdkRoot, "node_modules", "@connectrpc", "connect-node")), - ).toBe(false); fs.mkdirSync(blueprintRoot, { recursive: true }); fs.mkdirSync(privateRoot, { recursive: true }); @@ -406,7 +313,6 @@ describe("packaged Blueprint Runner external target", () => { privateAuthenticationPath, PRIVATE_AUTHENTICATION_CONTENTS, PRIVATE_AMBIENT_CONTENTS, - PRIVATE_ENTRY_SECRET, "BEGIN CERTIFICATE", ]; const runRunner = (argv: string[]) => { @@ -478,24 +384,6 @@ describe("packaged Blueprint Runner external target", () => { expectStableSingleLineDiagnostic(invalidAction.result.stderr); expect(invalidAction.evidence).toEqual({ effects: [] }); - const installedRunnerSource = fs.readFileSync(installedRuntimeRunner, "utf8"); - const boundedEntryDiagnostic = (() => { - try { - fs.copyFileSync(UNSAFE_RUNNER_FIXTURE, installedRuntimeRunner); - return runRunner(["status", "--external-target"]); - } finally { - fs.writeFileSync(installedRuntimeRunner, installedRunnerSource); - } - })(); - expect(boundedEntryDiagnostic.result.status, boundedEntryDiagnostic.safeDiagnostics).toBe( - 1, - ); - expect(boundedEntryDiagnostic.result.stderr).toContain(""); - expect(boundedEntryDiagnostic.result.stderr).not.toContain(PRIVATE_ENTRY_SECRET); - expect(boundedEntryDiagnostic.result.stderr.length).toBeLessThanOrEqual(1_032); - expectStableSingleLineDiagnostic(boundedEntryDiagnostic.result.stderr); - expect(boundedEntryDiagnostic.evidence).toEqual({ effects: [] }); - const installedSdk = packagePath(installRoot, OPENSHELL_SDK_PACKAGE); const disabledSdk = `${installedSdk}.disabled`; fs.renameSync(installedSdk, disabledSdk); @@ -583,8 +471,6 @@ describe("packaged Blueprint Runner external target", () => { invalidBlueprint.result.stderr, invalidAction.result.stdout, invalidAction.result.stderr, - boundedEntryDiagnostic.result.stdout, - boundedEntryDiagnostic.result.stderr, ].join("\n"); expect(privateValues.some((value) => publicOutput.includes(value))).toBe(false); } finally { diff --git a/test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts b/test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts deleted file mode 100644 index 427835d6d10..00000000000 --- a/test/package-contract/fixtures/blueprint-runner-unsafe-diagnostic.ts +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -const privateEntrySecret = `nvapi-${"A".repeat(64)}`; - -export async function main() { - throw new Error(`api_key=${privateEntrySecret}\u001b[31m\u202e\n${"x".repeat(5_000)}`); -} diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index 183961438e9..37d79a62655 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -29,6 +29,7 @@ export const CLI_ARTIFACT_UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; export const CLI_ARTIFACT_RESTORE_ACTION = E2E_ACTION_PROVENANCE.restoreCliArtifact.reference; export const CLI_ARTIFACT_PACKAGE_STEP = "Package exact-commit CLI"; +export const CLI_ARTIFACT_PACKAGE_SCRIPT = "scripts/e2e/package-cli-artifact.sh"; export const CLI_ARTIFACT_PUBLISH_STEP = "Publish content-addressed CLI artifact"; export const CLI_ARTIFACT_RESTORE_STEP = "Restore exact-commit CLI artifact"; @@ -285,7 +286,6 @@ function validateProducer(errors: string[], producer: WorkflowRecord): void { const uploadStep = uploadSteps[0]; const provenanceStep = provenanceSteps[0]; if (!packageStep || !uploadStep || !provenanceStep) return; - if (packageStep.id !== "package_cli_artifact" || packageStep.shell !== "bash") { errors.push("CLI artifact package step must use id package_cli_artifact and the Bash shell"); } @@ -302,40 +302,11 @@ function validateProducer(errors: string[], producer: WorkflowRecord): void { "CLI artifact package step must bind candidate and trusted workflow identities explicitly", ); } - requireFragments(errors, "CLI artifact package step", packageStep.run, [ - 'git rev-parse --verify HEAD)" == "$CANDIDATE_SHA"', - "dist/nemoclaw.js", - "dist/build-identity.json", - "dist/lib/blueprint-runner.js", - "dist/nemoclaw/package.json", - "dist/nemoclaw/blueprint/runner.js", - '[[ -f "$required_file" && ! -L "$required_file" && -s "$required_file" ]]', - "openshell-gateway-health-sdk.js", - "openshell-observation-boundary.cjs", - "sandbox-name.cjs", - '[[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]]', - 'artifact_catalog="dist/e2e-managed-image-catalog.json"', - '[[ ! -e "$artifact_catalog" && ! -L "$artifact_catalog" ]]', - "candidate build created the managed-image catalog path", - - ".sourceRevision == $candidateSha", - "candidate CLI build identity does not match the candidate commit SHA", - '(keys | sort) == ["type"]', - '.type == "module"', - "candidate Blueprint Runner module boundary is invalid", - "--sort=name", - "--mtime=@0", - "nemoclaw/dist/shared", - - "source_tree=\"$(git rev-parse 'HEAD^{tree}')\"", - 'lockfile_sha256="$(sha256sum package-lock.json', - 'artifact_name="nemoclaw-cli-${CANDIDATE_SHA}-${payload_sha256}"', - 'kind: "nemoclaw-e2e-cli-artifact-v1"', - "sha: $candidateSha", - "sha: $workflowSha", - "sourceRevision: $candidateSha", - "sha256: $payloadSha256", - ]); + if (packageStep.run !== CLI_ARTIFACT_PACKAGE_SCRIPT) { + errors.push( + `CLI artifact package step must invoke ${CLI_ARTIFACT_PACKAGE_SCRIPT} directly`, + ); + } if ( uploadStep.id !== "upload_cli_artifact" || From 1cdac432dc8e37628127429c1c185f1a62bd5ea8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Mon, 31 Aug 2026 19:18:54 -0700 Subject: [PATCH 21/28] refactor(openshell): narrow gateway health ownership Signed-off-by: Apurv Kumaria --- .../adapters/openshell/sandbox-observer.ts | 24 -- src/lib/blueprint-runner.ts | 6 +- ...l-gateway-health-workflow-boundary.test.ts | 249 ++++++++++++++++++ .../blueprint-external-target-plan.test.ts | 9 - 4 files changed, 251 insertions(+), 37 deletions(-) diff --git a/src/lib/adapters/openshell/sandbox-observer.ts b/src/lib/adapters/openshell/sandbox-observer.ts index b701bc59447..b2f1cebb359 100644 --- a/src/lib/adapters/openshell/sandbox-observer.ts +++ b/src/lib/adapters/openshell/sandbox-observer.ts @@ -76,27 +76,3 @@ export function namedOpenShellGateway(gatewayName: string): OpenShellGatewayTarg export function selectedOpenShellGateway(): OpenShellGatewayTarget { return { kind: "selected" }; } - -// sourceOfTruth: nemoclaw/src/shared/openshell-observation-boundary.cts -// generatedBoundary: build:cli emits this typed capability before compiling -// the root adapter surface. -export { - EXTERNAL_OPENSHELL_RELEASE, - observeExternalOpenShellGatewayHealth, -} from "../../../../nemoclaw/dist/shared/openshell-observation-boundary.cjs"; - -export { - createOpenShellSdkGatewayHealthObserver, - sdkOpenShellGatewayHealthObserver, -} from "../../../../nemoclaw/dist/shared/openshell-gateway-health-sdk.js"; - -export type { - ExternalOpenShellGatewayResult, - ExternalOpenShellGatewayStatus, - ObserveOpenShellGatewayHealthRequest, - OpenShellGatewayHealthError, - OpenShellGatewayHealthObservation, - OpenShellGatewayHealthObserver, - OpenShellGatewayHealthResult, - OpenShellGatewayHealthStatus, -} from "../../../../nemoclaw/dist/shared/openshell-observation-boundary.cjs"; diff --git a/src/lib/blueprint-runner.ts b/src/lib/blueprint-runner.ts index 0491cefff92..2c00ad9a8e8 100755 --- a/src/lib/blueprint-runner.ts +++ b/src/lib/blueprint-runner.ts @@ -3,10 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 import { redactForLog } from "./security/redact.js"; -import { - sdkOpenShellGatewayHealthObserver, - type OpenShellGatewayHealthObserver, -} from "./adapters/openshell/sandbox-observer.js"; +import { sdkOpenShellGatewayHealthObserver } from "../../nemoclaw/dist/shared/openshell-gateway-health-sdk.js"; +import type { OpenShellGatewayHealthObserver } from "../../nemoclaw/dist/shared/openshell-observation-boundary.cjs"; const RUNNER_MODULE = "../nemoclaw/blueprint/runner.js"; const MAX_DIAGNOSTIC_CHARACTERS = 1_024; diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts index c8435b13885..7d8682464d5 100644 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts @@ -6,8 +6,203 @@ import type { ChildProcess } from "node:child_process"; import { describe, expect, it, vi } from "vitest"; +import { + type ExternalGatewayHealthWorkflow, + validateExternalGatewayHealthWorkflow, +} from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; +import { PREPARE_E2E_ACTION } from "../../../tools/e2e/prepare-e2e-workflow-boundary.mts"; +import { UPLOAD_E2E_ARTIFACTS_ACTION } from "../../../tools/e2e/upload-e2e-artifacts-workflow-boundary.mts"; import { stopExternalGatewayHealthGateway } from "../fixtures/external-gateway-health-process.ts"; +const CHECKOUT_ACTION = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; +const SETUP_NODE_ACTION = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; +const DOWNLOAD_ACTION = "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; +const UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; +const VALID_HELPER_SOURCE = `const BLUEPRINT_RUNNER = path.join( + import.meta.dirname, + "..", + "..", + "..", + "dist", + "lib", + "blueprint-runner.js", +); +const address = externalHostAddress(); +const config = \`bind_address = "\${address}:\${String(port)}"\`; +const args = [BLUEPRINT_RUNNER, "status", "--external-target"]; +const certificateArgs = ["--server-san", address]; +const result = await shellProbe.run( + trustedShellCommand({ + command: process.execPath, + args: [BLUEPRINT_RUNNER, "status", "--external-target"], + captureLimitBytes: 64 * 1024, + env: { NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, + redactionValues: [privateStateRoot], + }), +); +const retry = { + operation: "external-gateway-health.tcp-readiness", + maxAttempts: 10, +}; +artifacts.addRedactionValues([stateDir]); +const receipt = { runner: "dist/lib/blueprint-runner.js" }; +`; + +function validWorkflow(): ExternalGatewayHealthWorkflow { + return { + jobs: { + "package-openshell-sdk": { + if: "${{ github.event_name == 'workflow_dispatch' && contains(format(',{0},', inputs.jobs), ',external-gateway-health,') }}", + "runs-on": "ubuntu-latest", + "timeout-minutes": 5, + permissions: { contents: "read", packages: "read" }, + outputs: { artifact_name: "${{ steps.identity.outputs.artifact_name }}" }, + steps: [ + { + name: "Check out trusted OpenShell SDK package verifier", + uses: CHECKOUT_ACTION, + with: { + ref: "${{ github.workflow_sha }}", + "persist-credentials": false, + "sparse-checkout": [ + "ci/reviewed-npm-audit.json", + "scripts/audit-reviewed-npm-graph.mts", + "scripts/checks/package-openshell-sdk-for-pr.mts", + "scripts/lib/openclaw-npm-remediation.mts", + "scripts/lib/reviewed-npm-archive.mts", + "scripts/lib/reviewed-npm-audit.mts", + ].join("\n"), + "sparse-checkout-cone-mode": false, + }, + }, + { + name: "Set up Node for reviewed package download", + uses: SETUP_NODE_ACTION, + with: { + "node-version": "22", + "registry-url": "https://npm.pkg.github.com", + scope: "@nvidia", + }, + }, + { + id: "package", + name: "Download and verify exact OpenShell SDK package", + env: { + NEMOCLAW_OPEN_SHELL_SDK_OUTPUT_DIRECTORY: "${{ runner.temp }}/openshell-sdk", + NODE_AUTH_TOKEN: "${{ github.token }}", + }, + run: "node --experimental-strip-types scripts/checks/package-openshell-sdk-for-pr.mts", + }, + { + id: "identity", + name: "Record reviewed OpenShell SDK artifact identity", + env: { + RUN_ATTEMPT: "${{ github.run_attempt }}", + RUN_ID: "${{ github.run_id }}", + }, + run: 'artifact_name="openshell-sdk-e2e-${RUN_ID}-${RUN_ATTEMPT}"', + }, + { + name: "Upload reviewed OpenShell SDK archive", + uses: UPLOAD_ACTION, + with: { + name: "${{ steps.identity.outputs.artifact_name }}", + path: "${{ steps.package.outputs.artifact_path }}", + "if-no-files-found": "error", + "retention-days": 1, + }, + }, + ], + }, + "external-gateway-health": { + needs: ["generate-matrix", "package-openshell-sdk"], + if: "${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'external-gateway-health') }}", + "runs-on": "ubuntu-latest", + "timeout-minutes": 15, + env: { + E2E_AGENT_RUNTIME: "none", + E2E_ARTIFACT_DIR: + "${{ github.workspace }}/e2e-artifacts/live/external-gateway-health", + E2E_DEFAULT_ENABLED: "0", + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: + "Ubuntu host with OpenShell 0.0.106; no inference endpoint", + E2E_JOB: "1", + E2E_OBSERVABLE_OUTCOME: + "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA", + E2E_TARGET_ID: "external-gateway-health", + NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106", + NEMOCLAW_RUN_LIVE_E2E: "1", + }, + steps: [ + { + name: "Check out candidate", + uses: CHECKOUT_ACTION, + with: { + repository: "${{ inputs.checkout_repository || github.repository }}", + ref: "${{ inputs.checkout_sha || github.sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }, + }, + { + name: "Prepare E2E workspace", + uses: PREPARE_E2E_ACTION, + with: { "build-cli": "false" }, + }, + { + name: "Restore exact-commit CLI artifact", + uses: + "NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + with: { + "provenance-json": + "${{ needs.generate-matrix.outputs.cli_artifact_provenance }}", + }, + }, + { + name: "Download reviewed OpenShell SDK archive", + uses: DOWNLOAD_ACTION, + with: { + name: "${{ needs.package-openshell-sdk.outputs.artifact_name }}", + path: "${{ runner.temp }}/openshell-sdk", + }, + }, + { + name: "Install reviewed OpenShell SDK archive without package credentials", + run: 'env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN npm install --no-save --package-lock=false --ignore-scripts "${archives[0]}"', + }, + { + name: "Install OpenShell CLI", + run: "env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN bash scripts/install-openshell.sh", + }, + { + name: "Run external gateway health live test", + run: "node tools/e2e/live-vitest-invocation.mts run test/e2e/live/external-gateway-health.test.ts", + }, + { + name: "Upload external gateway health artifacts", + if: "always()", + uses: UPLOAD_E2E_ARTIFACTS_ACTION, + with: { + name: "e2e-external-gateway-health", + path: "e2e-artifacts/live/external-gateway-health/", + }, + }, + ], + }, + }, + }; +} + +function healthStep(workflow: ExternalGatewayHealthWorkflow, name: string) { + const step = workflow.jobs["external-gateway-health"]!.steps!.find( + (candidate) => candidate.name === name, + ); + expect(step, `synthetic health step '${name}' is missing`).toBeDefined(); + return step!; +} + describe("external gateway health workflow boundary", () => { type FakeGateway = EventEmitter & { exitCode: number | null; @@ -32,6 +227,60 @@ describe("external gateway health workflow boundary", () => { return { gateway: gateway as ChildProcess, kill }; } + it("accepts the credential-free reviewed SDK workflow boundary (#9872)", () => { + expect(validateExternalGatewayHealthWorkflow(validWorkflow(), VALID_HELPER_SOURCE)).toEqual([]); + }); + + it("rejects a job-scoped package credential (#9872)", () => { + const workflow = validWorkflow(); + workflow.jobs["external-gateway-health"]!.env!.NODE_AUTH_TOKEN = "untrusted"; + + expect(validateExternalGatewayHealthWorkflow(workflow, VALID_HELPER_SOURCE)).toContain( + "external-gateway-health must not expose NODE_AUTH_TOKEN at job scope", + ); + }); + + it("rejects a credential-bearing reviewed SDK install (#9872)", () => { + const workflow = validWorkflow(); + healthStep( + workflow, + "Install reviewed OpenShell SDK archive without package credentials", + ).run = 'npm install "${archives[0]}"'; + + expect(validateExternalGatewayHealthWorkflow(workflow, VALID_HELPER_SOURCE)).toContain( + "external-gateway-health SDK install must retain: env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN", + ); + }); + + it("rejects a helper that bypasses the exact Runner status command (#9872)", () => { + const helper = VALID_HELPER_SOURCE.replaceAll( + '[BLUEPRINT_RUNNER, "status", "--external-target"]', + '[BLUEPRINT_RUNNER, "status"]', + ); + + expect(validateExternalGatewayHealthWorkflow(validWorkflow(), helper)).toContain( + "external gateway health helper must run exact Blueprint Runner external status", + ); + }); + + it("rejects SDK installation after the health test (#9872)", () => { + const workflow = validWorkflow(); + const steps = workflow.jobs["external-gateway-health"]!.steps!; + const installIndex = steps.findIndex( + (step) => + step.name === "Install reviewed OpenShell SDK archive without package credentials", + ); + const [install] = steps.splice(installIndex, 1); + const runIndex = steps.findIndex( + (step) => step.name === "Run external gateway health live test", + ); + steps.splice(runIndex + 1, 0, install!); + + expect(validateExternalGatewayHealthWorkflow(workflow, VALID_HELPER_SOURCE)).toContain( + "external-gateway-health step 'Install reviewed OpenShell SDK archive without package credentials' must precede 'Run external gateway health live test'", + ); + }); + it("waits for forced gateway exit before cleanup completes (#9872)", async () => { vi.useFakeTimers(); try { diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index 6def5e7c057..81960e3dec0 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -303,14 +303,6 @@ describe("packaged Blueprint Runner external target", () => { "blueprint", "runner.js", ); - const installedRootObserver = path.join( - installedPackage, - "dist", - "lib", - "adapters", - "openshell", - "sandbox-observer.js", - ); const installedSharedSdkObserver = path.join( installedPackage, "nemoclaw", @@ -332,7 +324,6 @@ describe("packaged Blueprint Runner external target", () => { "nemoclaw-blueprint-runner", ); expect(fs.existsSync(installedRunner)).toBe(true); - expect(fs.existsSync(installedRootObserver)).toBe(true); expect(fs.existsSync(installedSharedSdkObserver)).toBe(true); expect(fs.existsSync(installedRuntimeSdkObserver)).toBe(false); expect(fs.existsSync(path.join(installedPackage, "nemoclaw", "src"))).toBe(false); From b1ae72e611456104b221ef69d4b11e615910399c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 31 Aug 2026 19:47:32 -0700 Subject: [PATCH 22/28] docs(openshell): cite external target trust decision Signed-off-by: Carlos Villela --- .../openshell-typescript-sdk-0.0.106-dependency-review.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md b/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md index 42ddc30e3b4..55fb843ce2e 100644 --- a/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md +++ b/internal/security-reviews/openshell-typescript-sdk-0.0.106-dependency-review.md @@ -117,8 +117,9 @@ The image software bill of materials and third-party notice must bind to the ins - Severity and confidence: high, high confidence. - Failure mode: a validated hostname resolves or rebinds to a local service when the request starts. -- Control: the accepted initial contract treats cluster and storage administrators as trusted - infrastructure actors. The configured endpoint and CA come from those administrators. The +- Control: the [accepted read-only slice decision](https://github.com/NVIDIA/NemoClaw/issues/9872#issuecomment-5417049313) + treats cluster and storage administrators as trusted infrastructure actors. The configured + endpoint and CA come from those administrators. The operation sends only unauthenticated public health and retains TLS certificate and hostname verification. - Residual: SDK 0.0.106 does not expose a pinned lookup. A hostname can resolve differently between From dcd34b9781d9b0192a06cd0b8a3c1c9118b4d839 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 31 Aug 2026 20:06:00 -0700 Subject: [PATCH 23/28] fix(openshell): complete advisor repairs Signed-off-by: Carlos Villela --- .../blueprint/runner-external-target.test.ts | 41 +- nemoclaw/src/blueprint/runner.ts | 5 + scripts/find-source-shape-tests.mts | 4 - test/e2e/mock-parity.json | 2 +- .../external-gateway-health-process.test.ts | 76 ++++ ...l-gateway-health-workflow-boundary.test.ts | 325 ---------------- test/repository/source-shape-scanner.test.ts | 28 -- ...ernal-gateway-health-workflow-boundary.mts | 368 ------------------ tools/e2e/workflow-boundary.mts | 7 - 9 files changed, 118 insertions(+), 738 deletions(-) create mode 100644 test/e2e/support/external-gateway-health-process.test.ts delete mode 100644 test/e2e/support/external-gateway-health-workflow-boundary.test.ts delete mode 100644 tools/e2e/external-gateway-health-workflow-boundary.mts diff --git a/nemoclaw/src/blueprint/runner-external-target.test.ts b/nemoclaw/src/blueprint/runner-external-target.test.ts index dd9fcecb2d0..ea616028ff5 100644 --- a/nemoclaw/src/blueprint/runner-external-target.test.ts +++ b/nemoclaw/src/blueprint/runner-external-target.test.ts @@ -19,6 +19,7 @@ const externalTargetBoundaryMocks = vi.hoisted(() => ({ buildSanitizedExternalOpenShellTargetPlan: vi.fn(), withExternalOpenShellTargetCa: vi.fn(), })); +const fsMocks = vi.hoisted(() => ({ readFileSync: vi.fn() })); const observeHealth = vi.fn(); const gatewayHealthObserver = { observeHealth }; @@ -35,7 +36,7 @@ vi.mock("node:fs", async (importOriginal) => { return { ...original, mkdirSync: memory.mkdirSync, - readFileSync: memory.readFileSync, + readFileSync: fsMocks.readFileSync.mockImplementation(memory.readFileSync), writeFileSync: memory.writeFileSync, readdirSync: memory.readdirSync, }; @@ -125,14 +126,17 @@ describe("Blueprint Runner external OpenShell target", () => { expect(() => actionExternalOpenShellTargetPlan({})).toThrow( /does not declare an external OpenShell target/, ); - expect(() => actionExternalOpenShellTargetPlan({ openshell_target: {} })).toThrow( - /requires blueprint min_openshell_version and max_openshell_version/, - ); + expect(() => + actionExternalOpenShellTargetPlan({ version: "1.0.0", openshell_target: {} }), + ).toThrow(/requires blueprint min_openshell_version and max_openshell_version/); await expect(actionExternalOpenShellTargetStatus({}, gatewayHealthObserver)).rejects.toThrow( /does not declare an external OpenShell target/, ); await expect( - actionExternalOpenShellTargetStatus({ openshell_target: {} }, gatewayHealthObserver), + actionExternalOpenShellTargetStatus( + { version: "1.0.0", openshell_target: {} }, + gatewayHealthObserver, + ), ).rejects.toThrow(/requires blueprint min_openshell_version and max_openshell_version/); expect( @@ -142,6 +146,33 @@ describe("Blueprint Runner external OpenShell target", () => { expect(observeHealth).not.toHaveBeenCalled(); }); + it.each([ + ["missing", undefined, ["plan"]], + ["invalid", "1.0", ["plan"]], + ["missing", undefined, ["status", "--external-target"]], + ["invalid", "1.0", ["status", "--external-target"]], + ])( + "rejects a %s blueprint version for %s before target reads or observation", + async (_case, version, argv) => { + const blueprint = externalTargetBlueprint(); + blueprint.version = version; + addFile("blueprint.yaml", YAML.stringify(blueprint)); + + const action = argv[0] === "plan" ? "planning" : "status"; + await expect(runMain(argv)).rejects.toThrow( + `External OpenShell target ${action} requires blueprint version in X.Y.Z format.`, + ); + + expect(fsMocks.readFileSync).toHaveBeenCalledOnce(); + expect(fsMocks.readFileSync).toHaveBeenCalledWith("blueprint.yaml", "utf-8"); + expect( + externalTargetBoundaryMocks.buildSanitizedExternalOpenShellTargetPlan, + ).not.toHaveBeenCalled(); + expect(externalTargetBoundaryMocks.withExternalOpenShellTargetCa).not.toHaveBeenCalled(); + expect(observeHealth).not.toHaveBeenCalled(); + }, + ); + it("emits only the sanitized plan without subprocess or network calls (#9872)", async () => { vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://ambient-gateway.invalid"); vi.stubEnv("NEMOCLAW_GATEWAY_MANAGEMENT", "/private/ambient-gateway-management.json"); diff --git a/nemoclaw/src/blueprint/runner.ts b/nemoclaw/src/blueprint/runner.ts index a8c4952612d..364ee389626 100644 --- a/nemoclaw/src/blueprint/runner.ts +++ b/nemoclaw/src/blueprint/runner.ts @@ -1399,6 +1399,11 @@ function validateExternalOpenShellTargetBlueprint( if (blueprint.openshell_target === undefined) { throw new Error("blueprint does not declare an external OpenShell target"); } + if (blueprint.version === undefined || !/^[0-9]+\.[0-9]+\.[0-9]+$/.test(blueprint.version)) { + throw new Error( + `External OpenShell target ${action} requires blueprint version in X.Y.Z format.`, + ); + } if ( blueprint.min_openshell_version === undefined || blueprint.max_openshell_version === undefined diff --git a/scripts/find-source-shape-tests.mts b/scripts/find-source-shape-tests.mts index 18b0291b153..7bdcda7814b 100755 --- a/scripts/find-source-shape-tests.mts +++ b/scripts/find-source-shape-tests.mts @@ -588,10 +588,6 @@ function collectDeclarativeImports(imports: readonly ImportBinding[]): Map> = { - "tools/e2e/external-gateway-health-workflow-boundary": [ - "readExternalGatewayHealthHelper", - "readExternalGatewayHealthWorkflow", - ], "test/helpers/e2e-workflow-contract": ["readWorkflow", "readYaml"], "test/e2e/registry/registry": ["getTarget", "listTargets", "requireTargets"], "test/e2e/registry/expected-states": [ diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 89f92d5b4a7..ed102362d3c 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -239,7 +239,7 @@ "live": "test/e2e/live/external-gateway-health.test.ts", "liveSources": ["test/e2e/live/external-gateway-health-helpers.ts"], "fast": [ - "test/e2e/support/external-gateway-health-workflow-boundary.test.ts", + "test/e2e/support/external-gateway-health-process.test.ts", "test/install/reviewed-npm-archive.test.ts" ] }, diff --git a/test/e2e/support/external-gateway-health-process.test.ts b/test/e2e/support/external-gateway-health-process.test.ts new file mode 100644 index 00000000000..eb2df2d2153 --- /dev/null +++ b/test/e2e/support/external-gateway-health-process.test.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; + +import { describe, expect, it, vi } from "vitest"; + +import { stopExternalGatewayHealthGateway } from "../fixtures/external-gateway-health-process.ts"; + +type FakeGateway = EventEmitter & { + exitCode: number | null; + signalCode: NodeJS.Signals | null; + kill: ChildProcess["kill"]; +}; + +function gatewayWithForcedKill(onForcedKill: (gateway: FakeGateway) => void): { + gateway: ChildProcess; + kill: ReturnType>; +} { + const gateway = new EventEmitter() as FakeGateway; + gateway.exitCode = null; + gateway.signalCode = null; + const kill = vi.fn(); + kill.mockReturnValueOnce(true); + kill.mockImplementationOnce(() => { + onForcedKill(gateway); + return true; + }); + gateway.kill = kill; + return { gateway: gateway as ChildProcess, kill }; +} + +describe("external gateway health process cleanup", () => { + it("waits for forced gateway exit before cleanup completes (#9872)", async () => { + vi.useFakeTimers(); + try { + const { gateway, kill } = gatewayWithForcedKill((fakeGateway) => { + setTimeout(() => { + fakeGateway.signalCode = "SIGKILL"; + fakeGateway.emit("exit", null, "SIGKILL"); + }, 50); + }); + let completed = false; + const cleanup = stopExternalGatewayHealthGateway(gateway).then(() => { + completed = true; + }); + + await vi.advanceTimersByTimeAsync(2_000); + expect(kill).toHaveBeenNthCalledWith(1, "SIGTERM"); + expect(kill).toHaveBeenNthCalledWith(2, "SIGKILL"); + expect(completed).toBe(false); + + await vi.advanceTimersByTimeAsync(50); + await cleanup; + expect(completed).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it("reports a gateway that remains active after forced cleanup (#9872)", async () => { + vi.useFakeTimers(); + try { + const { gateway } = gatewayWithForcedKill(() => undefined); + const cleanup = expect(stopExternalGatewayHealthGateway(gateway)).rejects.toThrow( + "external gateway health gateway did not stop after SIGKILL", + ); + + await vi.advanceTimersByTimeAsync(4_000); + await cleanup; + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts b/test/e2e/support/external-gateway-health-workflow-boundary.test.ts deleted file mode 100644 index 7d8682464d5..00000000000 --- a/test/e2e/support/external-gateway-health-workflow-boundary.test.ts +++ /dev/null @@ -1,325 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { EventEmitter } from "node:events"; -import type { ChildProcess } from "node:child_process"; - -import { describe, expect, it, vi } from "vitest"; - -import { - type ExternalGatewayHealthWorkflow, - validateExternalGatewayHealthWorkflow, -} from "../../../tools/e2e/external-gateway-health-workflow-boundary.mts"; -import { PREPARE_E2E_ACTION } from "../../../tools/e2e/prepare-e2e-workflow-boundary.mts"; -import { UPLOAD_E2E_ARTIFACTS_ACTION } from "../../../tools/e2e/upload-e2e-artifacts-workflow-boundary.mts"; -import { stopExternalGatewayHealthGateway } from "../fixtures/external-gateway-health-process.ts"; - -const CHECKOUT_ACTION = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; -const SETUP_NODE_ACTION = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; -const DOWNLOAD_ACTION = "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; -const UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; -const VALID_HELPER_SOURCE = `const BLUEPRINT_RUNNER = path.join( - import.meta.dirname, - "..", - "..", - "..", - "dist", - "lib", - "blueprint-runner.js", -); -const address = externalHostAddress(); -const config = \`bind_address = "\${address}:\${String(port)}"\`; -const args = [BLUEPRINT_RUNNER, "status", "--external-target"]; -const certificateArgs = ["--server-san", address]; -const result = await shellProbe.run( - trustedShellCommand({ - command: process.execPath, - args: [BLUEPRINT_RUNNER, "status", "--external-target"], - captureLimitBytes: 64 * 1024, - env: { NEMOCLAW_BLUEPRINT_PATH: blueprintRoot }, - redactionValues: [privateStateRoot], - }), -); -const retry = { - operation: "external-gateway-health.tcp-readiness", - maxAttempts: 10, -}; -artifacts.addRedactionValues([stateDir]); -const receipt = { runner: "dist/lib/blueprint-runner.js" }; -`; - -function validWorkflow(): ExternalGatewayHealthWorkflow { - return { - jobs: { - "package-openshell-sdk": { - if: "${{ github.event_name == 'workflow_dispatch' && contains(format(',{0},', inputs.jobs), ',external-gateway-health,') }}", - "runs-on": "ubuntu-latest", - "timeout-minutes": 5, - permissions: { contents: "read", packages: "read" }, - outputs: { artifact_name: "${{ steps.identity.outputs.artifact_name }}" }, - steps: [ - { - name: "Check out trusted OpenShell SDK package verifier", - uses: CHECKOUT_ACTION, - with: { - ref: "${{ github.workflow_sha }}", - "persist-credentials": false, - "sparse-checkout": [ - "ci/reviewed-npm-audit.json", - "scripts/audit-reviewed-npm-graph.mts", - "scripts/checks/package-openshell-sdk-for-pr.mts", - "scripts/lib/openclaw-npm-remediation.mts", - "scripts/lib/reviewed-npm-archive.mts", - "scripts/lib/reviewed-npm-audit.mts", - ].join("\n"), - "sparse-checkout-cone-mode": false, - }, - }, - { - name: "Set up Node for reviewed package download", - uses: SETUP_NODE_ACTION, - with: { - "node-version": "22", - "registry-url": "https://npm.pkg.github.com", - scope: "@nvidia", - }, - }, - { - id: "package", - name: "Download and verify exact OpenShell SDK package", - env: { - NEMOCLAW_OPEN_SHELL_SDK_OUTPUT_DIRECTORY: "${{ runner.temp }}/openshell-sdk", - NODE_AUTH_TOKEN: "${{ github.token }}", - }, - run: "node --experimental-strip-types scripts/checks/package-openshell-sdk-for-pr.mts", - }, - { - id: "identity", - name: "Record reviewed OpenShell SDK artifact identity", - env: { - RUN_ATTEMPT: "${{ github.run_attempt }}", - RUN_ID: "${{ github.run_id }}", - }, - run: 'artifact_name="openshell-sdk-e2e-${RUN_ID}-${RUN_ATTEMPT}"', - }, - { - name: "Upload reviewed OpenShell SDK archive", - uses: UPLOAD_ACTION, - with: { - name: "${{ steps.identity.outputs.artifact_name }}", - path: "${{ steps.package.outputs.artifact_path }}", - "if-no-files-found": "error", - "retention-days": 1, - }, - }, - ], - }, - "external-gateway-health": { - needs: ["generate-matrix", "package-openshell-sdk"], - if: "${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'external-gateway-health') }}", - "runs-on": "ubuntu-latest", - "timeout-minutes": 15, - env: { - E2E_AGENT_RUNTIME: "none", - E2E_ARTIFACT_DIR: - "${{ github.workspace }}/e2e-artifacts/live/external-gateway-health", - E2E_DEFAULT_ENABLED: "0", - E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: - "Ubuntu host with OpenShell 0.0.106; no inference endpoint", - E2E_JOB: "1", - E2E_OBSERVABLE_OUTCOME: - "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA", - E2E_TARGET_ID: "external-gateway-health", - NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106", - NEMOCLAW_RUN_LIVE_E2E: "1", - }, - steps: [ - { - name: "Check out candidate", - uses: CHECKOUT_ACTION, - with: { - repository: "${{ inputs.checkout_repository || github.repository }}", - ref: "${{ inputs.checkout_sha || github.sha }}", - "fetch-depth": 0, - "persist-credentials": false, - }, - }, - { - name: "Prepare E2E workspace", - uses: PREPARE_E2E_ACTION, - with: { "build-cli": "false" }, - }, - { - name: "Restore exact-commit CLI artifact", - uses: - "NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - with: { - "provenance-json": - "${{ needs.generate-matrix.outputs.cli_artifact_provenance }}", - }, - }, - { - name: "Download reviewed OpenShell SDK archive", - uses: DOWNLOAD_ACTION, - with: { - name: "${{ needs.package-openshell-sdk.outputs.artifact_name }}", - path: "${{ runner.temp }}/openshell-sdk", - }, - }, - { - name: "Install reviewed OpenShell SDK archive without package credentials", - run: 'env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN npm install --no-save --package-lock=false --ignore-scripts "${archives[0]}"', - }, - { - name: "Install OpenShell CLI", - run: "env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN bash scripts/install-openshell.sh", - }, - { - name: "Run external gateway health live test", - run: "node tools/e2e/live-vitest-invocation.mts run test/e2e/live/external-gateway-health.test.ts", - }, - { - name: "Upload external gateway health artifacts", - if: "always()", - uses: UPLOAD_E2E_ARTIFACTS_ACTION, - with: { - name: "e2e-external-gateway-health", - path: "e2e-artifacts/live/external-gateway-health/", - }, - }, - ], - }, - }, - }; -} - -function healthStep(workflow: ExternalGatewayHealthWorkflow, name: string) { - const step = workflow.jobs["external-gateway-health"]!.steps!.find( - (candidate) => candidate.name === name, - ); - expect(step, `synthetic health step '${name}' is missing`).toBeDefined(); - return step!; -} - -describe("external gateway health workflow boundary", () => { - type FakeGateway = EventEmitter & { - exitCode: number | null; - signalCode: NodeJS.Signals | null; - kill: ChildProcess["kill"]; - }; - - function gatewayWithForcedKill(onForcedKill: (gateway: FakeGateway) => void): { - gateway: ChildProcess; - kill: ReturnType>; - } { - const gateway = new EventEmitter() as FakeGateway; - gateway.exitCode = null; - gateway.signalCode = null; - const kill = vi.fn(); - kill.mockReturnValueOnce(true); - kill.mockImplementationOnce(() => { - onForcedKill(gateway); - return true; - }); - gateway.kill = kill; - return { gateway: gateway as ChildProcess, kill }; - } - - it("accepts the credential-free reviewed SDK workflow boundary (#9872)", () => { - expect(validateExternalGatewayHealthWorkflow(validWorkflow(), VALID_HELPER_SOURCE)).toEqual([]); - }); - - it("rejects a job-scoped package credential (#9872)", () => { - const workflow = validWorkflow(); - workflow.jobs["external-gateway-health"]!.env!.NODE_AUTH_TOKEN = "untrusted"; - - expect(validateExternalGatewayHealthWorkflow(workflow, VALID_HELPER_SOURCE)).toContain( - "external-gateway-health must not expose NODE_AUTH_TOKEN at job scope", - ); - }); - - it("rejects a credential-bearing reviewed SDK install (#9872)", () => { - const workflow = validWorkflow(); - healthStep( - workflow, - "Install reviewed OpenShell SDK archive without package credentials", - ).run = 'npm install "${archives[0]}"'; - - expect(validateExternalGatewayHealthWorkflow(workflow, VALID_HELPER_SOURCE)).toContain( - "external-gateway-health SDK install must retain: env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN", - ); - }); - - it("rejects a helper that bypasses the exact Runner status command (#9872)", () => { - const helper = VALID_HELPER_SOURCE.replaceAll( - '[BLUEPRINT_RUNNER, "status", "--external-target"]', - '[BLUEPRINT_RUNNER, "status"]', - ); - - expect(validateExternalGatewayHealthWorkflow(validWorkflow(), helper)).toContain( - "external gateway health helper must run exact Blueprint Runner external status", - ); - }); - - it("rejects SDK installation after the health test (#9872)", () => { - const workflow = validWorkflow(); - const steps = workflow.jobs["external-gateway-health"]!.steps!; - const installIndex = steps.findIndex( - (step) => - step.name === "Install reviewed OpenShell SDK archive without package credentials", - ); - const [install] = steps.splice(installIndex, 1); - const runIndex = steps.findIndex( - (step) => step.name === "Run external gateway health live test", - ); - steps.splice(runIndex + 1, 0, install!); - - expect(validateExternalGatewayHealthWorkflow(workflow, VALID_HELPER_SOURCE)).toContain( - "external-gateway-health step 'Install reviewed OpenShell SDK archive without package credentials' must precede 'Run external gateway health live test'", - ); - }); - - it("waits for forced gateway exit before cleanup completes (#9872)", async () => { - vi.useFakeTimers(); - try { - const { gateway, kill } = gatewayWithForcedKill((fakeGateway) => { - setTimeout(() => { - fakeGateway.signalCode = "SIGKILL"; - fakeGateway.emit("exit", null, "SIGKILL"); - }, 50); - }); - let completed = false; - const cleanup = stopExternalGatewayHealthGateway(gateway).then(() => { - completed = true; - }); - - await vi.advanceTimersByTimeAsync(2_000); - expect(kill).toHaveBeenNthCalledWith(1, "SIGTERM"); - expect(kill).toHaveBeenNthCalledWith(2, "SIGKILL"); - expect(completed).toBe(false); - - await vi.advanceTimersByTimeAsync(50); - await cleanup; - expect(completed).toBe(true); - } finally { - vi.useRealTimers(); - } - }); - - it("reports a gateway that remains active after forced cleanup (#9872)", async () => { - vi.useFakeTimers(); - try { - const { gateway } = gatewayWithForcedKill(() => undefined); - const cleanup = expect(stopExternalGatewayHealthGateway(gateway)).rejects.toThrow( - "external gateway health gateway did not stop after SIGKILL", - ); - - await vi.advanceTimersByTimeAsync(4_000); - await cleanup; - } finally { - vi.useRealTimers(); - } - }); -}); diff --git a/test/repository/source-shape-scanner.test.ts b/test/repository/source-shape-scanner.test.ts index f6cda57f4dd..740eb4710c8 100644 --- a/test/repository/source-shape-scanner.test.ts +++ b/test/repository/source-shape-scanner.test.ts @@ -533,34 +533,6 @@ describe("source-shape scanner", () => { expect(cases).toEqual(["mirrors a namespace-loaded workflow"]); }); - it("detects an explicit assertion on the shipped external gateway helper source", () => { - const cases = detectedCaseNames(` - import { expect, it } from "vitest"; - import { readExternalGatewayHealthHelper } from "../tools/e2e/external-gateway-health-workflow-boundary.mts"; - - it("asserts the protected Runner boundary", () => { - const source = readExternalGatewayHealthHelper(); - expect(source).toContain("shellProbe.run"); - }); - `); - - expect(cases).toEqual(["asserts the protected Runner boundary"]); - }); - - it("detects an explicit assertion on the shipped external gateway workflow", () => { - const cases = detectedCaseNames(` - import { expect, it } from "vitest"; - import { readExternalGatewayHealthWorkflow } from "../tools/e2e/external-gateway-health-workflow-boundary.mts"; - - it("asserts the protected package job", () => { - const workflow = readExternalGatewayHealthWorkflow(); - expect(workflow.jobs).toHaveProperty("package-openshell-sdk"); - }); - `); - - expect(cases).toEqual(["asserts the protected package job"]); - }); - it("does not taint spawn results when raw config feeds the command", () => { const cases = detectedCaseNames(` import { spawnSync } from "node:child_process"; diff --git a/tools/e2e/external-gateway-health-workflow-boundary.mts b/tools/e2e/external-gateway-health-workflow-boundary.mts deleted file mode 100644 index 6f4a2dc4c1f..00000000000 --- a/tools/e2e/external-gateway-health-workflow-boundary.mts +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - -import YAML from "yaml"; -import { PREPARE_E2E_ACTION } from "./prepare-e2e-workflow-boundary.mts"; -import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-boundary.mts"; - -const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); -const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const DEFAULT_LIVE_HELPER_PATH = join( - REPO_ROOT, - "test", - "e2e", - "live", - "external-gateway-health-helpers.ts", -); -const PACKAGE_JOB = "package-openshell-sdk"; -const HEALTH_JOB = "external-gateway-health"; -const CHECKOUT_ACTION = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; -const SETUP_NODE_ACTION = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; -const DOWNLOAD_ACTION = "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; -const UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; - -type WorkflowStep = { - env?: Record; - id?: string; - if?: string; - name?: string; - run?: string; - uses?: string; - with?: Record; -}; - -type WorkflowJob = { - env?: Record; - if?: string; - needs?: string | string[]; - outputs?: Record; - permissions?: Record; - "runs-on"?: string; - steps?: WorkflowStep[]; - "timeout-minutes"?: number; -}; - -export type ExternalGatewayHealthWorkflow = { - jobs: Record; -}; - -export function readExternalGatewayHealthWorkflow( - workflowPath = DEFAULT_WORKFLOW_PATH, -): ExternalGatewayHealthWorkflow { - return YAML.parse(readFileSync(workflowPath, "utf8")) as ExternalGatewayHealthWorkflow; -} - -export function readExternalGatewayHealthHelper(helperPath = DEFAULT_LIVE_HELPER_PATH): string { - return readFileSync(helperPath, "utf8"); -} - -function findStep(job: WorkflowJob, name: string): WorkflowStep { - return job.steps?.find((step) => step.name === name) ?? {}; -} - -function sameMembers(value: string | string[] | undefined, expected: readonly string[]): boolean { - const actual = Array.isArray(value) ? value : value ? [value] : []; - return JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort()); -} - -function requireOrder( - errors: string[], - steps: WorkflowStep[], - beforeName: string, - afterName: string, -): void { - const before = steps.findIndex((step) => step.name === beforeName); - const after = steps.findIndex((step) => step.name === afterName); - if (before < 0 || after < 0 || before >= after) { - errors.push(`${HEALTH_JOB} step '${beforeName}' must precede '${afterName}'`); - } -} - -function validatePackageJob(errors: string[], job: WorkflowJob): void { - if ( - job.if !== - "${{ github.event_name == 'workflow_dispatch' && contains(format(',{0},', inputs.jobs), ',external-gateway-health,') }}" - ) { - errors.push(`${PACKAGE_JOB} must run only for the explicit external health selector`); - } - if ( - job["runs-on"] !== "ubuntu-latest" || - job["timeout-minutes"] !== 5 || - JSON.stringify(job.permissions) !== JSON.stringify({ contents: "read", packages: "read" }) - ) { - errors.push(`${PACKAGE_JOB} must retain its bounded package-read trust boundary`); - } - if (job.outputs?.artifact_name !== "${{ steps.identity.outputs.artifact_name }}") { - errors.push(`${PACKAGE_JOB} must expose only the recorded artifact name`); - } - - const steps = job.steps ?? []; - const checkout = findStep(job, "Check out trusted OpenShell SDK package verifier"); - const sparse = String(checkout.with?.["sparse-checkout"] ?? "") - .split(/\r?\n/u) - .map((entry) => entry.trim()) - .filter(Boolean) - .sort(); - const expectedSparse = [ - "ci/reviewed-npm-audit.json", - "scripts/audit-reviewed-npm-graph.mts", - "scripts/checks/package-openshell-sdk-for-pr.mts", - "scripts/lib/openclaw-npm-remediation.mts", - "scripts/lib/reviewed-npm-archive.mts", - "scripts/lib/reviewed-npm-audit.mts", - ].sort(); - if ( - checkout.uses !== CHECKOUT_ACTION || - checkout.with?.ref !== "${{ github.workflow_sha }}" || - checkout.with?.["persist-credentials"] !== false || - checkout.with?.["sparse-checkout-cone-mode"] !== false || - JSON.stringify(sparse) !== JSON.stringify(expectedSparse) - ) { - errors.push(`${PACKAGE_JOB} must execute only the trusted sparse package verifier checkout`); - } - - const setup = findStep(job, "Set up Node for reviewed package download"); - if ( - setup.uses !== SETUP_NODE_ACTION || - setup.with?.["node-version"] !== "22" || - setup.with?.["registry-url"] !== "https://npm.pkg.github.com" || - setup.with?.scope !== "@nvidia" - ) { - errors.push(`${PACKAGE_JOB} must use the reviewed Node and GitHub Packages setup`); - } - - const download = findStep(job, "Download and verify exact OpenShell SDK package"); - if ( - download.id !== "package" || - download.env?.NEMOCLAW_OPEN_SHELL_SDK_OUTPUT_DIRECTORY !== "${{ runner.temp }}/openshell-sdk" || - download.env?.NODE_AUTH_TOKEN !== "${{ github.token }}" || - !download.run?.includes( - "node --experimental-strip-types scripts/checks/package-openshell-sdk-for-pr.mts", - ) - ) { - errors.push(`${PACKAGE_JOB} must scope its package credential to the reviewed downloader`); - } - - const identity = findStep(job, "Record reviewed OpenShell SDK artifact identity"); - if ( - identity.id !== "identity" || - identity.env?.RUN_ATTEMPT !== "${{ github.run_attempt }}" || - identity.env?.RUN_ID !== "${{ github.run_id }}" || - !identity.run?.includes('artifact_name="openshell-sdk-e2e-${RUN_ID}-${RUN_ATTEMPT}"') - ) { - errors.push(`${PACKAGE_JOB} must bind the artifact name to this workflow attempt`); - } - - const upload = findStep(job, "Upload reviewed OpenShell SDK archive"); - if ( - upload.uses !== UPLOAD_ACTION || - upload.with?.name !== "${{ steps.identity.outputs.artifact_name }}" || - upload.with?.path !== "${{ steps.package.outputs.artifact_path }}" || - upload.with?.["if-no-files-found"] !== "error" || - upload.with?.["retention-days"] !== 1 - ) { - errors.push(`${PACKAGE_JOB} must publish only the exact one-day reviewed archive`); - } - requireOrder(errors, steps, checkout.name ?? "", setup.name ?? ""); - requireOrder(errors, steps, setup.name ?? "", download.name ?? ""); - requireOrder(errors, steps, download.name ?? "", upload.name ?? ""); -} - -function validateHealthJob(errors: string[], job: WorkflowJob): void { - if (!sameMembers(job.needs, ["generate-matrix", PACKAGE_JOB])) { - errors.push(`${HEALTH_JOB} must wait for the candidate CLI and reviewed SDK archive`); - } - if ( - job.if !== - "${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'external-gateway-health') }}" - ) { - errors.push(`${HEALTH_JOB} must use the trusted explicit selection and dependency gate`); - } - if (job["runs-on"] !== "ubuntu-latest" || job["timeout-minutes"] !== 15) { - errors.push(`${HEALTH_JOB} must retain its 15 minute Ubuntu resource budget`); - } - const env = job.env ?? {}; - const expectedEnv = { - E2E_AGENT_RUNTIME: "none", - E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/external-gateway-health", - E2E_DEFAULT_ENABLED: "0", - E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: - "Ubuntu host with OpenShell 0.0.106; no inference endpoint", - E2E_JOB: "1", - E2E_NON_INTERACTIVE: undefined, - E2E_OBSERVABLE_OUTCOME: - "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA", - E2E_TARGET_ID: HEALTH_JOB, - NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.106", - NEMOCLAW_RUN_LIVE_E2E: "1", - }; - for (const [name, value] of Object.entries(expectedEnv)) { - if (value === undefined ? Object.hasOwn(env, name) : env[name] !== value) { - errors.push(`${HEALTH_JOB} must retain ${name}=${String(value)}`); - } - } - for (const name of ["GITHUB_TOKEN", "NODE_AUTH_TOKEN", "NVIDIA_API_KEY"]) { - if (Object.hasOwn(env, name)) errors.push(`${HEALTH_JOB} must not expose ${name} at job scope`); - } - - const steps = job.steps ?? []; - const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; - if ( - checkout.uses !== CHECKOUT_ACTION || - checkout.with?.repository !== "${{ inputs.checkout_repository || github.repository }}" || - checkout.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" || - checkout.with?.["fetch-depth"] !== 0 || - checkout.with?.["persist-credentials"] !== false - ) { - errors.push( - `${HEALTH_JOB} must use the exact candidate checkout without persisted credentials`, - ); - } - const prepare = findStep(job, "Prepare E2E workspace"); - if (prepare.uses !== PREPARE_E2E_ACTION || prepare.with?.["build-cli"] !== "false") { - errors.push(`${HEALTH_JOB} must use the reviewed dependency preparation without rebuilding`); - } - const restore = findStep(job, "Restore exact-commit CLI artifact"); - if ( - !restore.uses?.startsWith("NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@") || - restore.with?.["provenance-json"] !== - "${{ needs.generate-matrix.outputs.cli_artifact_provenance }}" - ) { - errors.push(`${HEALTH_JOB} must restore the exact candidate CLI artifact`); - } - const download = findStep(job, "Download reviewed OpenShell SDK archive"); - if ( - download.uses !== DOWNLOAD_ACTION || - download.with?.name !== "${{ needs.package-openshell-sdk.outputs.artifact_name }}" || - download.with?.path !== "${{ runner.temp }}/openshell-sdk" - ) { - errors.push(`${HEALTH_JOB} must download only this run's reviewed SDK archive`); - } - const installSdk = findStep( - job, - "Install reviewed OpenShell SDK archive without package credentials", - ); - for (const fragment of [ - "env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN", - 'npm install --no-save --package-lock=false --ignore-scripts "${archives[0]}"', - ]) { - if (!installSdk.run?.includes(fragment)) { - errors.push(`${HEALTH_JOB} SDK install must retain: ${fragment}`); - } - } - const installOpenShell = findStep(job, "Install OpenShell CLI"); - if ( - installOpenShell.run !== - "env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN bash scripts/install-openshell.sh" - ) { - errors.push(`${HEALTH_JOB} OpenShell install must not receive package credentials`); - } - const run = findStep(job, "Run external gateway health live test"); - if ( - !run.run?.includes("tools/e2e/live-vitest-invocation.mts run") || - !run.run?.includes("test/e2e/live/external-gateway-health.test.ts") || - Object.keys(run.env ?? {}).length > 0 || - JSON.stringify(run).includes("secrets.") - ) { - errors.push(`${HEALTH_JOB} must run only the credential-free external health test`); - } - const upload = findStep(job, "Upload external gateway health artifacts"); - if ( - upload.if !== "always()" || - upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || - upload.with?.name !== "e2e-external-gateway-health" || - upload.with?.path !== "e2e-artifacts/live/external-gateway-health/" - ) { - errors.push(`${HEALTH_JOB} must always use the reviewed artifact uploader`); - } - requireOrder(errors, steps, prepare.name ?? "", restore.name ?? ""); - requireOrder(errors, steps, restore.name ?? "", download.name ?? ""); - requireOrder(errors, steps, download.name ?? "", installSdk.name ?? ""); - requireOrder(errors, steps, installSdk.name ?? "", run.name ?? ""); - requireOrder(errors, steps, run.name ?? "", upload.name ?? ""); -} - -export function validateExternalGatewayHealthHelper(source: string): string[] { - const errors: string[] = []; - const exactRunnerPath = `const BLUEPRINT_RUNNER = path.join( - import.meta.dirname, - "..", - "..", - "..", - "dist", - "lib", - "blueprint-runner.js", -);`; - const required = [ - exactRunnerPath, - '[BLUEPRINT_RUNNER, "status", "--external-target"]', - `const result = await shellProbe.run( - trustedShellCommand({ - command: process.execPath,`, - '"status", "--external-target"', - "captureLimitBytes: 64 * 1024", - "NEMOCLAW_BLUEPRINT_PATH: blueprintRoot", - "redactionValues: [privateStateRoot]", - ]; - if (required.some((fragment) => !source.includes(fragment))) { - errors.push("external gateway health helper must run exact Blueprint Runner external status"); - } - if ( - !source.includes("const address = externalHostAddress();") || - !source.includes('bind_address = "${address}:${String(port)}"') || - !source.includes('"--server-san", address') - ) { - errors.push( - "external gateway health helper must bind the gateway certificate to a non-loopback address", - ); - } - if ( - !source.includes('operation: "external-gateway-health.tcp-readiness"') || - !source.includes("maxAttempts: 10,") - ) { - errors.push("external gateway health helper must retain the bounded readiness retry"); - } - if (!source.includes('runner: "dist/lib/blueprint-runner.js"')) { - errors.push("external gateway health helper must record the exact Runner artifact identity"); - } - if (!source.includes("artifacts.addRedactionValues([stateDir]);")) { - errors.push("external gateway health helper must redact its private fixture path"); - } - if (source.includes("@nvidia/openshell-sdk")) { - errors.push( - "external gateway health helper must not bypass the Runner with a direct SDK import", - ); - } - return errors; -} - -export function validateExternalGatewayHealthWorkflow( - workflow: ExternalGatewayHealthWorkflow, - liveHelperSource = readExternalGatewayHealthHelper(), -): string[] { - const errors: string[] = []; - const packageJob = workflow.jobs[PACKAGE_JOB]; - const healthJob = workflow.jobs[HEALTH_JOB]; - if (!packageJob) errors.push(`workflow is missing ${PACKAGE_JOB}`); - else validatePackageJob(errors, packageJob); - if (!healthJob) errors.push(`workflow is missing ${HEALTH_JOB}`); - else validateHealthJob(errors, healthJob); - errors.push(...validateExternalGatewayHealthHelper(liveHelperSource)); - return errors; -} - -export function validateExternalGatewayHealthWorkflowBoundary( - workflowPath = DEFAULT_WORKFLOW_PATH, - liveHelperPath = DEFAULT_LIVE_HELPER_PATH, -): string[] { - return validateExternalGatewayHealthWorkflow( - readExternalGatewayHealthWorkflow(workflowPath), - readExternalGatewayHealthHelper(liveHelperPath), - ); -} diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index f62dd67153e..3fe8353cbd4 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -55,10 +55,6 @@ import { validateUploadE2eArtifactsWorkflowBoundary, } from "./upload-e2e-artifacts-workflow-boundary.mts"; import { validateE2eWorkspaceBootstrapBoundary } from "./workspace-bootstrap-workflow-boundary.mts"; -import { - type ExternalGatewayHealthWorkflow, - validateExternalGatewayHealthWorkflow, -} from "./external-gateway-health-workflow-boundary.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_E2E_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); @@ -2610,9 +2606,6 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { ...validateOpenShellGatewayAuthContractWorkflow( workflow as unknown as OpenShellGatewayAuthContractWorkflow, ), - ...validateExternalGatewayHealthWorkflow( - workflow as unknown as ExternalGatewayHealthWorkflow, - ), ); errors.push(...validateE2eOperationsWorkflow(workflow as unknown as OperationsWorkflow)); errors.push(...validateStandardProfileWorkflowBoundary(workflow)); From 0587cd555d98963aa2f9f61ab22759029337fe41 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 31 Aug 2026 20:29:59 -0700 Subject: [PATCH 24/28] chore: apply review repair formatting Signed-off-by: Carlos Villela --- .../blueprint/runner-external-target.test.ts | 25 + nemoclaw/src/blueprint/runner.ts | 13 +- .../cli-artifact-workflow-boundary.test.ts | 271 +------- test/helpers/vitest-watch-triggers.ts | 1 - .../blueprint-external-target-plan.test.ts | 630 ++++++++++-------- test/repository/vitest-watch-triggers.test.ts | 1 - tools/e2e/cli-artifact-workflow-boundary.mts | 561 ---------------- .../workspace-bootstrap-workflow-boundary.mts | 10 +- 8 files changed, 388 insertions(+), 1124 deletions(-) diff --git a/nemoclaw/src/blueprint/runner-external-target.test.ts b/nemoclaw/src/blueprint/runner-external-target.test.ts index ea616028ff5..0b0e6e08bd0 100644 --- a/nemoclaw/src/blueprint/runner-external-target.test.ts +++ b/nemoclaw/src/blueprint/runner-external-target.test.ts @@ -173,6 +173,31 @@ describe("Blueprint Runner external OpenShell target", () => { }, ); + it.each([ + ["planning", ["plan"]], + ["status", ["status", "--external-target"]], + ])( + "rejects an unknown top-level field before external target %s reads or observation", + async (_action, argv) => { + addFile( + "blueprint.yaml", + YAML.stringify({ ...externalTargetBlueprint(), unknown_top_level_field: true }), + ); + + await expect(runMain(argv)).rejects.toThrow( + "blueprint.yaml must contain a YAML mapping with valid nested component shapes", + ); + + expect(fsMocks.readFileSync).toHaveBeenCalledOnce(); + expect(fsMocks.readFileSync).toHaveBeenCalledWith("blueprint.yaml", "utf-8"); + expect( + externalTargetBoundaryMocks.buildSanitizedExternalOpenShellTargetPlan, + ).not.toHaveBeenCalled(); + expect(externalTargetBoundaryMocks.withExternalOpenShellTargetCa).not.toHaveBeenCalled(); + expect(observeHealth).not.toHaveBeenCalled(); + }, + ); + it("emits only the sanitized plan without subprocess or network calls (#9872)", async () => { vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://ambient-gateway.invalid"); vi.stubEnv("NEMOCLAW_GATEWAY_MANAGEMENT", "/private/ambient-gateway-management.json"); diff --git a/nemoclaw/src/blueprint/runner.ts b/nemoclaw/src/blueprint/runner.ts index 364ee389626..988826804ce 100644 --- a/nemoclaw/src/blueprint/runner.ts +++ b/nemoclaw/src/blueprint/runner.ts @@ -169,6 +169,17 @@ const HTTP_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", " const REST_PROTOCOLS = new Set(["rest"]); const ENDPOINT_ENFORCEMENT_MODES = new Set(["enforce", "audit"]); const ENDPOINT_TLS_MODES = new Set(["terminate", "passthrough", "skip"]); +const BLUEPRINT_KEYS = new Set([ + "version", + "min_openshell_version", + "max_openshell_version", + "min_openclaw_version", + "digest", + "profiles", + "description", + "openshell_target", + "components", +]); const MISSING_PROVIDER_INSPECTION_PATTERN = /(?:\bprovider\b[^\r\n]*\b(?:not found|does not exist)\b|\b(?:not found|does not exist)\b[^\r\n]*\bprovider\b|\bunknown provider\b)/i; const POLICY_INSPECTION_MAX_BYTES = 1024 * 1024; @@ -354,7 +365,7 @@ function isInferenceProfile(value: unknown): value is InferenceProfile { } function isBlueprint(value: unknown): value is Blueprint { - if (!isPlainObject(value)) { + if (!isPlainObject(value) || !Object.keys(value).every((key) => BLUEPRINT_KEYS.has(key))) { return false; } diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index fd7cc1044af..774a6a7d33a 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -7,20 +7,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { - CLI_ARTIFACT_PACKAGE_SCRIPT, - CLI_ARTIFACT_PACKAGE_STEP, - CLI_ARTIFACT_PUBLISH_STEP, - validateCliArtifactRestoreAction, - validateCliArtifactWorkflowBoundary, -} from "../../../tools/e2e/cli-artifact-workflow-boundary.mts"; -import { - type CompositeAction, - readRepoText, - readWorkflow, - readYaml, - type Workflow, -} from "../../helpers/e2e-workflow-contract"; +import { type CompositeAction, readYaml } from "../../helpers/e2e-workflow-contract"; const CANDIDATE_SHA = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8", @@ -65,10 +52,6 @@ function runIdentityValidation(overrides: Record = {}, consumer } } -function workflowFixture(): Workflow { - return JSON.parse(JSON.stringify(readWorkflow())) as Workflow; -} - type RestoreFixtureOptions = { archive?: | "valid" @@ -448,188 +431,7 @@ function expectRestoreFailure(options: RestoreFixtureOptions, message: string): } } -function requireStep(workflow: Workflow, jobName: string, stepName: string) { - const step = workflow.jobs[jobName]?.steps?.find((candidate) => candidate.name === stepName); - expect(step, `${jobName} must contain ${stepName}`).toBeDefined(); - return step!; -} - -describe("exact-commit CLI artifact workflow boundary", () => { - it("builds the candidate CLI once and requires every artifact-using job to restore it", () => { - expect(validateCliArtifactWorkflowBoundary(readWorkflow())).toEqual([]); - }); - - it("rejects dependency caching before trusted installation (#9051)", () => { - const workflow = workflowFixture(); - const setupNode = requireStep( - workflow, - "mcp-bridge-dev", - "Set up Node.js for trusted OpenShell verification", - ); - setupNode.with = { ...setupNode.with, "package-manager-cache": true }; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must set up Node.js without dependency caching before candidate checkout", - ); - }); - - it("rejects package manager probes after candidate checkout (#9051)", () => { - const workflow = workflowFixture(); - const steps = workflow.jobs["mcp-bridge-dev"].steps!; - const setupNode = requireStep( - workflow, - "mcp-bridge-dev", - "Set up Node.js for trusted OpenShell verification", - ); - const [movedSetupNode] = steps.splice(steps.indexOf(setupNode), 1); - const candidateCheckoutIndex = steps.findIndex((step) => - step.uses?.startsWith("actions/checkout@"), - ); - steps.splice(candidateCheckoutIndex + 1, 0, movedSetupNode!); - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must set up Node.js without dependency caching before candidate checkout", - ); - }); - - it("rejects candidate execution before trusted development installation (#9051)", () => { - const workflow = workflowFixture(); - const steps = workflow.jobs["mcp-bridge-dev"].steps!; - const candidateCheckoutIndex = steps.findIndex((step) => - step.uses?.startsWith("actions/checkout@"), - ); - steps.splice(candidateCheckoutIndex + 1, 0, { - name: "Execute candidate CLI before trusted installation", - run: "node bin/nemoclaw.js --version", - }); - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must preserve every reviewed step through trusted installation", - ); - }); - - it("rejects candidate execution embedded in the trusted development installer (#9051)", () => { - const workflow = workflowFixture(); - const install = requireStep( - workflow, - "mcp-bridge-dev", - "Install immutable OpenShell dev artifact", - ); - install.run = `bash test/e2e/setup-mcp-test-tls.sh\n${install.run ?? ""}`; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must preserve every reviewed step through trusted installation", - ); - }); - - it("rejects candidate-controlled process hooks in the trusted development job (#9051)", () => { - const workflow = workflowFixture(); - const job = workflow.jobs["mcp-bridge-dev"]; - expect(job).toBeDefined(); - job!.env = { - ...job!.env, - NODE_OPTIONS: "--require=./candidate-preload.cjs", - }; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must not use candidate-controlled process hooks before trusted installation", - ); - }); - - it("rejects candidate-controlled process hooks in the workflow environment (#9051)", () => { - const workflow = workflowFixture() as Workflow & { env?: Record }; - workflow.env = { - ...workflow.env, - NODE_OPTIONS: "--require=./candidate-preload.cjs", - }; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "workflow must not set process startup hooks before CLI artifact restore", - ); - }); - - it("scans the complete job for process hooks when trusted installation is missing (#9051)", () => { - const workflow = workflowFixture(); - const job = workflow.jobs["mcp-bridge-dev"]; - const steps = job.steps!; - job.steps = steps.filter( - (step) => step.name !== "Install immutable OpenShell dev artifact", - ); - const prepare = requireStep(workflow, "mcp-bridge-dev", "Prepare E2E workspace"); - prepare.env = { NODE_OPTIONS: "--require=./candidate-preload.cjs" }; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toEqual( - expect.arrayContaining([ - "mcp-bridge-dev must not use candidate-controlled process hooks before trusted installation", - "mcp-bridge-dev must preserve every reviewed step through trusted installation", - ]), - ); - }); - - it("rejects skipping the trusted development installer (#9051)", () => { - const workflow = workflowFixture(); - const install = requireStep( - workflow, - "mcp-bridge-dev", - "Install immutable OpenShell dev artifact", - ); - install.if = "${{ false }}"; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must preserve every reviewed step through trusted installation", - ); - }); - - it("rejects skipping post-install dependency preparation (#9051)", () => { - const workflow = workflowFixture(); - const prepare = requireStep(workflow, "mcp-bridge-dev", "Prepare E2E workspace"); - prepare.if = "${{ false }}"; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must preserve reviewed dependency preparation and candidate CLI restore after trusted installation", - ); - }); - - it("rejects removing post-install dependency preparation and candidate CLI restore (#9051)", () => { - const workflow = workflowFixture(); - const steps = workflow.jobs["mcp-bridge-dev"].steps!; - workflow.jobs["mcp-bridge-dev"].steps = steps.filter( - (step) => - step.name !== "Prepare E2E workspace" && - step.name !== "Restore exact-commit CLI artifact", - ); - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "mcp-bridge-dev must verify and restore the exact CLI artifact exactly once", - ); - }); - - it("rejects removing the trusted development consumer job (#9051)", () => { - const workflow = workflowFixture(); - delete workflow.jobs["mcp-bridge-dev"]; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toContain( - "workflow is missing required CLI artifact consumer mcp-bridge-dev", - ); - }); - - it("reports both an unreadable action and a missing producer", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "cli-artifact-missing-action-")); - try { - const workflow = workflowFixture(); - delete workflow.jobs["generate-matrix"]; - - expect( - validateCliArtifactWorkflowBoundary(workflow, path.join(directory, "missing-action.yaml")), - ).toEqual([ - "CLI artifact restore action file is missing or unreadable", - "workflow is missing CLI artifact producer generate-matrix", - ]); - } finally { - fs.rmSync(directory, { force: true, recursive: true }); - } - }); - +describe("exact-commit CLI artifact restore", () => { it("accepts matching artifact, candidate source, and workflow identities", () => { const result = runIdentityValidation(); expect(result.status, "matching artifact identity validation failed").toBe(0); @@ -900,73 +702,4 @@ describe("exact-commit CLI artifact workflow boundary", () => { "restored CLI build identity does not match the candidate SHA", ); }); - - it("rejects producer identity and content-addressing drift", () => { - const workflow = workflowFixture(); - const producer = workflow.jobs["generate-matrix"]; - producer.outputs!.cli_artifact_provenance = - "${{ steps.upload_cli_artifact.outputs.artifact-url }}"; - const packageStep = requireStep(workflow, "generate-matrix", CLI_ARTIFACT_PACKAGE_STEP); - packageStep.env!.WORKFLOW_SHA = "${{ inputs.checkout_sha }}"; - packageStep.run = `bash ${CLI_ARTIFACT_PACKAGE_SCRIPT}`; - const uploadStep = requireStep(workflow, "generate-matrix", CLI_ARTIFACT_PUBLISH_STEP); - uploadStep.uses = "actions/upload-artifact@v7"; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toEqual( - expect.arrayContaining([ - "generate-matrix must expose exact cli_artifact_provenance provenance", - "CLI artifact package step must bind candidate and trusted workflow identities explicitly", - `CLI artifact package step must invoke ${CLI_ARTIFACT_PACKAGE_SCRIPT} directly`, - "CLI artifact upload must use the immutable content-addressed upload contract", - ]), - ); - }); - - it("leaves job timeouts to their dedicated workflow validators", () => { - const workflow = workflowFixture(); - const timeoutMinutes = Number(workflow.jobs["hermes-e2e"]["timeout-minutes"]); - expect(Number.isFinite(timeoutMinutes)).toBe(true); - workflow.jobs["hermes-e2e"]["timeout-minutes"] = timeoutMinutes + 1; - - expect(validateCliArtifactWorkflowBoundary(workflow)).toEqual([]); - }); - - it("rejects action implementation drift that weakens extraction or payload verification", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "cli-artifact-action-")); - try { - const actionPath = path.join(directory, "action.yaml"); - const source = readRepoText(".github/actions/restore-e2e-cli-artifact/action.yaml") - .replace("tar --no-same-owner --no-same-permissions", "tar") - .replace("sandbox-name.cjs", "missing-boundary.cjs") - .replace( - 'lockfile_sha256="$(sha256_file package-lock.json)"', - [ - '# lockfile_sha256="$(sha256_file package-lock.json)"', - 'lockfile_sha256="$(openssl dgst -sha256 -r package-lock.json | cut -d " " -f 1)"', - ].join("\n "), - ) - .replace( - 'actual_payload_sha256="$(sha256_file "$payload")"', - [ - '# actual_payload_sha256="$(sha256_file "$payload")"', - 'actual_payload_sha256="$(openssl dgst -sha256 -r "$payload" | cut -d " " -f 1)"', - ].join("\n "), - ) - .replace('[[ "$actual_payload_sha256" == "$PAYLOAD_SHA256" ]]', '[[ -s "$payload" ]]'); - fs.writeFileSync(actionPath, source); - - expect(validateCliArtifactRestoreAction(actionPath)).toEqual( - expect.arrayContaining([ - "CLI artifact restore action must match its immutable workflow pin", - 'CLI artifact payload verification must contain tar --no-same-owner --no-same-permissions -xf "$payload" -C "$restore_dir"', - "CLI artifact payload verification must contain sandbox-name.cjs", - "CLI artifact payload verification must assign lockfile_sha256 exactly once through the Node.js binary stream", - "CLI artifact payload verification must assign actual_payload_sha256 exactly once through the Node.js binary stream", - 'CLI artifact payload verification must contain [[ "$actual_payload_sha256" == "$PAYLOAD_SHA256" ]]', - ]), - ); - } finally { - fs.rmSync(directory, { force: true, recursive: true }); - } - }); }); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 7e618a4c143..6689594782a 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -8,7 +8,6 @@ export type VitestWatchTriggerPattern = { const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/base-image-publication-workflow-boundary.test.ts", - "test/e2e/support/cli-artifact-workflow-boundary.test.ts", "test/e2e/support/dcode-profile-import-gate-workflow-boundary.test.ts", "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts", "test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts", diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index dd80176448f..e3c32224c89 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -8,7 +8,7 @@ import os from "node:os"; import path from "node:path"; import { rootCertificates } from "node:tls"; -import { describe, expect, it } from "vitest"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import YAML from "yaml"; const REPOSITORY_ROOT = path.join(import.meta.dirname, "..", ".."); @@ -195,287 +195,351 @@ describe("packaged Blueprint Runner npm cache", () => { }); }); -describe("packaged Blueprint Runner external target", () => { - it( - "executes the installed package command and fails before effects when the SDK is absent (#9872)", - { timeout: 240_000 }, - () => { - const fixtureRoot = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-external-target-package-"), +type RunnerResult = Readonly<{ + evidence: ProbeEvidence; + result: SpawnSyncReturns; + safeDiagnostics: string; +}>; + +describe.sequential("packaged Blueprint Runner external target", () => { + let ambientRoot: string; + let blueprintFile: string; + let blueprintRoot: string; + let disabledSdk: string; + let evidencePath: string; + let fixtureRoot: string; + let installedBinary: string; + let installedSdk: string; + let privateAuthenticationPath: string; + let privateCaPath: string; + let privateValues: string[]; + let runRunner: (argv: string[]) => RunnerResult; + let validBlueprint: string; + let validBlueprintDocument: Record; + + beforeAll(() => { + fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-external-target-package-")); + const archiveRoot = path.join(fixtureRoot, "archive"); + const consumerRoot = path.join(fixtureRoot, "consumer"); + const installRoot = path.join(fixtureRoot, "install"); + const runtimeRoot = path.join(fixtureRoot, "runtime"); + blueprintRoot = path.join(runtimeRoot, "blueprint"); + const privateRoot = path.join(runtimeRoot, "private-inputs"); + ambientRoot = path.join(runtimeRoot, "ambient-home"); + privateCaPath = path.join(privateRoot, "private-ca.pem"); + privateAuthenticationPath = path.join(privateRoot, "private-authentication"); + evidencePath = path.join(runtimeRoot, "probe-evidence.json"); + + fs.mkdirSync(archiveRoot, { recursive: true }); + const pack = spawnSync( + "npm", + ["pack", "--ignore-scripts", "--silent", "--pack-destination", archiveRoot], + { cwd: REPOSITORY_ROOT, encoding: "utf8", env: npmEnvironment() }, + ); + assertCommandSucceeded(pack, "root package archive creation"); + const archives = fs.readdirSync(archiveRoot).filter((entry) => entry.endsWith(".tgz")); + expect(archives).toHaveLength(1); + const archive = path.join(archiveRoot, archives[0]!); + + fs.mkdirSync(installRoot, { recursive: true }); + const extract = spawnSync("tar", ["-xzf", archive, "--strip-components=1", "-C", installRoot], { + encoding: "utf8", + env: npmEnvironment(), + }); + assertCommandSucceeded(extract, "root package archive extraction"); + fs.copyFileSync( + path.join(REPOSITORY_ROOT, "package-lock.json"), + path.join(installRoot, "package-lock.json"), + ); + const install = spawnSync("npm", ["ci", "--ignore-scripts", "--offline", "--omit=dev"], { + cwd: installRoot, + encoding: "utf8", + env: npmEnvironment(), + }); + assertCommandSucceeded(install, "locked installation of the packed Runner graph"); + + fs.mkdirSync(consumerRoot, { recursive: true }); + fs.writeFileSync( + path.join(consumerRoot, "package.json"), + JSON.stringify({ name: "nemoclaw-blueprint-runner-consumer", private: true }), + ); + const consumerInstall = spawnSync( + "npm", + [ + "install", + "--ignore-scripts", + "--offline", + "--omit=dev", + "--no-save", + "--package-lock=false", + installRoot, + ], + { + cwd: consumerRoot, + encoding: "utf8", + env: { ...npmEnvironment(), NEMOCLAW_INSTALLING: "1" }, + }, + ); + assertCommandSucceeded(consumerInstall, "offline consumer installation of the packed Runner"); + + installedBinary = path.join(consumerRoot, "node_modules", ".bin", "nemoclaw-blueprint-runner"); + installedSdk = packagePath(consumerRoot, OPENSHELL_SDK_PACKAGE); + disabledSdk = `${installedSdk}.disabled`; + + fs.mkdirSync(blueprintRoot, { recursive: true }); + fs.mkdirSync(privateRoot, { recursive: true }); + fs.mkdirSync(path.join(ambientRoot, ".config", "openshell"), { recursive: true }); + fs.writeFileSync(privateCaPath, CA_PEM); + fs.writeFileSync(privateAuthenticationPath, PRIVATE_AUTHENTICATION_CONTENTS); + fs.writeFileSync( + path.join(ambientRoot, ".config", "openshell", "gateway.env"), + PRIVATE_AMBIENT_CONTENTS, + ); + blueprintFile = path.join(blueprintRoot, "blueprint.yaml"); + validBlueprintDocument = { + version: "1.0.0", + min_openshell_version: "0.0.106", + max_openshell_version: "0.0.106", + openshell_target: { + endpoint: "https://192.0.2.1:8443", + workspace: "default", + expected_release: "0.0.106", + lifecycle: "external", + trust: { ca_file: privateCaPath }, + authentication: { credential_file: privateAuthenticationPath }, + }, + }; + validBlueprint = YAML.stringify(validBlueprintDocument); + + const probePath = path.join(installRoot, "runtime-probe.mjs"); + writeRuntimeProbe(probePath); + privateValues = [ + fixtureRoot, + privateCaPath, + privateAuthenticationPath, + PRIVATE_AUTHENTICATION_CONTENTS, + PRIVATE_AMBIENT_CONTENTS, + "BEGIN CERTIFICATE", + ]; + runRunner = (argv: string[]) => { + fs.rmSync(evidencePath, { force: true }); + const result = spawnSync(installedBinary, argv, { + cwd: runtimeRoot, + encoding: "utf8", + env: { + ...process.env, + HOME: ambientRoot, + NODE_OPTIONS: `--import=${probePath}`, + NEMOCLAW_BLUEPRINT_PATH: blueprintRoot, + NEMOCLAW_TEST_AMBIENT_ROOT: ambientRoot, + NEMOCLAW_TEST_AUTHENTICATION_FILE: privateAuthenticationPath, + NEMOCLAW_TEST_EVIDENCE_FILE: evidencePath, + XDG_CONFIG_HOME: path.join(ambientRoot, ".config"), + }, + }); + const safeDiagnostics = privateValues.reduce( + (output, value) => output.replaceAll(value, "[redacted]"), + commandOutput(result), ); - const archiveRoot = path.join(fixtureRoot, "archive"); - const consumerRoot = path.join(fixtureRoot, "consumer"); - const installRoot = path.join(fixtureRoot, "install"); - const runtimeRoot = path.join(fixtureRoot, "runtime"); - const blueprintRoot = path.join(runtimeRoot, "blueprint"); - const privateRoot = path.join(runtimeRoot, "private-inputs"); - const ambientRoot = path.join(runtimeRoot, "ambient-home"); - const privateCaPath = path.join(privateRoot, "private-ca.pem"); - const privateAuthenticationPath = path.join(privateRoot, "private-authentication"); - const evidencePath = path.join(runtimeRoot, "probe-evidence.json"); - - try { - fs.mkdirSync(archiveRoot, { recursive: true }); - const pack = spawnSync( - "npm", - ["pack", "--ignore-scripts", "--silent", "--pack-destination", archiveRoot], - { cwd: REPOSITORY_ROOT, encoding: "utf8", env: npmEnvironment() }, - ); - assertCommandSucceeded(pack, "root package archive creation"); - const archives = fs.readdirSync(archiveRoot).filter((entry) => entry.endsWith(".tgz")); - expect(archives).toHaveLength(1); - const archive = path.join(archiveRoot, archives[0]!); - - fs.mkdirSync(installRoot, { recursive: true }); - const extract = spawnSync( - "tar", - ["-xzf", archive, "--strip-components=1", "-C", installRoot], - { encoding: "utf8", env: npmEnvironment() }, - ); - assertCommandSucceeded(extract, "root package archive extraction"); - fs.copyFileSync( - path.join(REPOSITORY_ROOT, "package-lock.json"), - path.join(installRoot, "package-lock.json"), - ); - const install = spawnSync("npm", ["ci", "--ignore-scripts", "--offline", "--omit=dev"], { - cwd: installRoot, - encoding: "utf8", - env: npmEnvironment(), - }); - assertCommandSucceeded(install, "locked installation of the packed Runner graph"); - - fs.mkdirSync(consumerRoot, { recursive: true }); - fs.writeFileSync( - path.join(consumerRoot, "package.json"), - JSON.stringify({ name: "nemoclaw-blueprint-runner-consumer", private: true }), - ); - const consumerInstall = spawnSync( - "npm", - [ - "install", - "--ignore-scripts", - "--offline", - "--omit=dev", - "--no-save", - "--package-lock=false", - installRoot, - ], - { - cwd: consumerRoot, - encoding: "utf8", - env: { ...npmEnvironment(), NEMOCLAW_INSTALLING: "1" }, - }, - ); - assertCommandSucceeded( - consumerInstall, - "offline consumer installation of the packed Runner", - ); - - const installedPackage = installRoot; - const installedBinary = path.join( - consumerRoot, - "node_modules", - ".bin", - "nemoclaw-blueprint-runner", - ); - - fs.mkdirSync(blueprintRoot, { recursive: true }); - fs.mkdirSync(privateRoot, { recursive: true }); - fs.mkdirSync(path.join(ambientRoot, ".config", "openshell"), { recursive: true }); - fs.writeFileSync(privateCaPath, CA_PEM); - fs.writeFileSync(privateAuthenticationPath, PRIVATE_AUTHENTICATION_CONTENTS); - fs.writeFileSync( - path.join(ambientRoot, ".config", "openshell", "gateway.env"), - PRIVATE_AMBIENT_CONTENTS, - ); - const blueprintFile = path.join(blueprintRoot, "blueprint.yaml"); - const validBlueprintDocument = { - version: "1.0.0", - min_openshell_version: "0.0.106", - max_openshell_version: "0.0.106", - openshell_target: { - endpoint: "https://192.0.2.1:8443", - workspace: "default", - expected_release: "0.0.106", - lifecycle: "external", - trust: { ca_file: privateCaPath }, - authentication: { credential_file: privateAuthenticationPath }, - }, - }; - const validBlueprint = YAML.stringify(validBlueprintDocument); - fs.writeFileSync(blueprintFile, validBlueprint); - - const probePath = path.join(installedPackage, "runtime-probe.mjs"); - writeRuntimeProbe(probePath); - const privateValues = [ - fixtureRoot, - privateCaPath, - privateAuthenticationPath, - PRIVATE_AUTHENTICATION_CONTENTS, - PRIVATE_AMBIENT_CONTENTS, - "BEGIN CERTIFICATE", - ]; - const runRunner = (argv: string[]) => { - fs.rmSync(evidencePath, { force: true }); - const result = spawnSync(installedBinary, argv, { - cwd: runtimeRoot, - encoding: "utf8", - env: { - ...process.env, - HOME: ambientRoot, - NODE_OPTIONS: `--import=${probePath}`, - NEMOCLAW_BLUEPRINT_PATH: blueprintRoot, - NEMOCLAW_TEST_AMBIENT_ROOT: ambientRoot, - NEMOCLAW_TEST_AUTHENTICATION_FILE: privateAuthenticationPath, - NEMOCLAW_TEST_EVIDENCE_FILE: evidencePath, - XDG_CONFIG_HOME: path.join(ambientRoot, ".config"), - }, - }); - const safeDiagnostics = privateValues.reduce( - (output, value) => output.replaceAll(value, "[redacted]"), - commandOutput(result), - ); - expect( - fs.existsSync(evidencePath), - `packaged runner did not write probe evidence:\n${safeDiagnostics}`, - ).toBe(true); - const evidence = JSON.parse(fs.readFileSync(evidencePath, "utf8")) as ProbeEvidence; - return { evidence, result, safeDiagnostics }; - }; - - const hiddenBlueprint = `${blueprintFile}.hidden`; - fs.renameSync(blueprintFile, hiddenBlueprint); - const missingBlueprint = runRunner(["status", "--external-target"]); - fs.renameSync(hiddenBlueprint, blueprintFile); - expect(missingBlueprint.result.status, missingBlueprint.safeDiagnostics).toBe(1); - expect(missingBlueprint.result.stderr).toContain("blueprint.yaml not found"); - expect(missingBlueprint.result.stderr).not.toContain(blueprintRoot); - expectStableSingleLineDiagnostic(missingBlueprint.result.stderr); - expect(missingBlueprint.evidence).toEqual({ effects: [] }); - - fs.writeFileSync( - blueprintFile, - `openshell_target:\n trust:\n ca_file: [${privateCaPath}\u001b[31m`, - ); - const malformedBlueprint = runRunner(["status", "--external-target"]); - expect(malformedBlueprint.result.status, malformedBlueprint.safeDiagnostics).toBe(1); - expect(malformedBlueprint.result.stderr).toContain("blueprint.yaml contains invalid YAML"); - expect(malformedBlueprint.result.stderr).not.toContain(privateCaPath); - expectStableSingleLineDiagnostic(malformedBlueprint.result.stderr); - expect(malformedBlueprint.evidence).toEqual({ effects: [] }); - - fs.writeFileSync(blueprintFile, privateCaPath); - const invalidBlueprint = runRunner(["status", "--external-target"]); - expect(invalidBlueprint.result.status, invalidBlueprint.safeDiagnostics).toBe(1); - expect(invalidBlueprint.result.stderr).toContain( - "blueprint.yaml must contain a YAML mapping with valid nested component shapes", - ); - expect(invalidBlueprint.result.stderr).not.toContain(privateCaPath); - expectStableSingleLineDiagnostic(invalidBlueprint.result.stderr); - expect(invalidBlueprint.evidence).toEqual({ effects: [] }); - - fs.writeFileSync(blueprintFile, validBlueprint); - const invalidAction = runRunner([`\u001b[31m${privateCaPath}\u202e`]); - expect(invalidAction.result.status, invalidAction.safeDiagnostics).toBe(1); - expect(invalidAction.result.stderr).toContain( - "Unknown action. Use: plan, apply, status, reconcile, rollback, snapshots", - ); - expect(invalidAction.result.stderr).not.toContain(privateCaPath); - expectStableSingleLineDiagnostic(invalidAction.result.stderr); - expect(invalidAction.evidence).toEqual({ effects: [] }); - - const installedSdk = packagePath(installRoot, OPENSHELL_SDK_PACKAGE); - const disabledSdk = `${installedSdk}.disabled`; - fs.renameSync(installedSdk, disabledSdk); - const absentStatus = runRunner(["status", "--external-target"]); - fs.renameSync(disabledSdk, installedSdk); - expect(absentStatus.result.status, absentStatus.safeDiagnostics).toBe(1); - expect(absentStatus.result.stderr).toContain( - "The approved OpenShell SDK 0.0.106 is unavailable.", - ); - expect(absentStatus.result.stderr).not.toContain("NemoClaw could not reach"); - expect(absentStatus.evidence).toEqual({ effects: [] }); - - const presentStatus = runRunner(["status", "--external-target"]); - expect(presentStatus.result.status, presentStatus.safeDiagnostics).toBe(1); - expect(presentStatus.result.stderr).toContain( - "NemoClaw could not reach the external OpenShell target.", - ); - expect(presentStatus.evidence).toEqual({ effects: ["network"] }); - - fs.writeFileSync( - blueprintFile, - YAML.stringify({ - ...validBlueprintDocument, - min_openshell_version: "0.0.105", - openshell_target: { - ...validBlueprintDocument.openshell_target, - expected_release: "0.0.105", - }, - }), - ); - const unsupportedReleasePlan = runRunner(["plan"]); - expect(unsupportedReleasePlan.result.status, unsupportedReleasePlan.safeDiagnostics).toBe( - 1, - ); - expect(unsupportedReleasePlan.result.stderr).toContain( - "external OpenShell target expected_release must be 0.0.106", - ); - expect(unsupportedReleasePlan.result.stdout).not.toContain("openshell_target"); - expect(unsupportedReleasePlan.evidence.effects).toEqual([]); - fs.writeFileSync(blueprintFile, validBlueprint); - - const plan = runRunner(["plan"]); - expect(plan.result.status, plan.safeDiagnostics).toBe(0); - expect(plan.evidence.effects).toEqual([]); - const planStart = plan.result.stdout.indexOf("{"); - expect(planStart).toBeGreaterThan(-1); - expect(JSON.parse(plan.result.stdout.slice(planStart))).toEqual({ - run_id: expect.stringMatching(/^nc-\d{8}-\d{6}-[0-9a-f]{8}$/u), - openshell_target: { - endpoint: "https://192.0.2.1:8443", - workspace: "default", - expected_release: "0.0.106", - lifecycle: "external", - authentication_source: "file", - ca_fingerprint: `sha256:${createHash("sha256") - .update(new X509Certificate(CA_PEM).raw) - .digest("hex")}`, - }, - dry_run: false, - }); - - const apply = runRunner(["apply"]); - expect(apply.result.status, apply.safeDiagnostics).toBe(1); - expect(apply.result.stderr).toContain( - "External OpenShell target apply is not available until typed readiness and inventory are implemented.", - ); - expect(apply.evidence.effects).toEqual([]); - - const publicOutput = [ - absentStatus.result.stdout, - absentStatus.result.stderr, - presentStatus.result.stdout, - presentStatus.result.stderr, - unsupportedReleasePlan.result.stdout, - unsupportedReleasePlan.result.stderr, - plan.result.stdout, - plan.result.stderr, - apply.result.stdout, - apply.result.stderr, - missingBlueprint.result.stdout, - missingBlueprint.result.stderr, - malformedBlueprint.result.stdout, - malformedBlueprint.result.stderr, - invalidBlueprint.result.stdout, - invalidBlueprint.result.stderr, - invalidAction.result.stdout, - invalidAction.result.stderr, - ].join("\n"); - expect(privateValues.some((value) => publicOutput.includes(value))).toBe(false); - } finally { - fs.rmSync(fixtureRoot, { recursive: true, force: true }); - } - }, - ); + expect( + fs.existsSync(evidencePath), + `packaged runner did not write probe evidence:\n${safeDiagnostics}`, + ).toBe(true); + const evidence = JSON.parse(fs.readFileSync(evidencePath, "utf8")) as ProbeEvidence; + return { evidence, result, safeDiagnostics }; + }; + }, 240_000); + + beforeEach(() => { + fs.mkdirSync(blueprintRoot, { recursive: true }); + fs.mkdirSync(path.dirname(privateCaPath), { recursive: true }); + fs.mkdirSync(path.join(ambientRoot, ".config", "openshell"), { recursive: true }); + fs.writeFileSync(privateCaPath, CA_PEM); + fs.writeFileSync(privateAuthenticationPath, PRIVATE_AUTHENTICATION_CONTENTS); + fs.writeFileSync( + path.join(ambientRoot, ".config", "openshell", "gateway.env"), + PRIVATE_AMBIENT_CONTENTS, + ); + fs.writeFileSync(blueprintFile, validBlueprint); + }); + + afterAll(() => { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + }); + + function expectPrivateValuesRedacted(result: SpawnSyncReturns): void { + expect(privateValues.some((value) => commandOutput(result).includes(value))).toBe(false); + } + + it("rejects a missing packaged blueprint before effects (#9872)", () => { + // Setup + const hiddenBlueprint = `${blueprintFile}.hidden`; + fs.renameSync(blueprintFile, hiddenBlueprint); + + // Action + const execution = runRunner(["status", "--external-target"]); + fs.renameSync(hiddenBlueprint, blueprintFile); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain("blueprint.yaml not found"); + expect(execution.result.stderr).not.toContain(blueprintRoot); + expectStableSingleLineDiagnostic(execution.result.stderr); + expect(execution.evidence).toEqual({ effects: [] }); + expectPrivateValuesRedacted(execution.result); + }); + + it("rejects malformed packaged blueprint YAML before effects (#9872)", () => { + // Setup + fs.writeFileSync( + blueprintFile, + `openshell_target:\n trust:\n ca_file: [${privateCaPath}\u001b[31m`, + ); + + // Action + const execution = runRunner(["status", "--external-target"]); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain("blueprint.yaml contains invalid YAML"); + expect(execution.result.stderr).not.toContain(privateCaPath); + expectStableSingleLineDiagnostic(execution.result.stderr); + expect(execution.evidence).toEqual({ effects: [] }); + expectPrivateValuesRedacted(execution.result); + }); + + it("rejects a packaged blueprint without a mapping before effects (#9872)", () => { + // Setup + fs.writeFileSync(blueprintFile, privateCaPath); + + // Action + const execution = runRunner(["status", "--external-target"]); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain( + "blueprint.yaml must contain a YAML mapping with valid nested component shapes", + ); + expect(execution.result.stderr).not.toContain(privateCaPath); + expectStableSingleLineDiagnostic(execution.result.stderr); + expect(execution.evidence).toEqual({ effects: [] }); + expectPrivateValuesRedacted(execution.result); + }); + + it("rejects an invalid packaged runner action before effects (#9872)", () => { + // Setup + const invalidAction = `\u001b[31m${privateCaPath}\u202e`; + + // Action + const execution = runRunner([invalidAction]); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain( + "Unknown action. Use: plan, apply, status, reconcile, rollback, snapshots", + ); + expect(execution.result.stderr).not.toContain(privateCaPath); + expectStableSingleLineDiagnostic(execution.result.stderr); + expect(execution.evidence).toEqual({ effects: [] }); + expectPrivateValuesRedacted(execution.result); + }); + + it("fails before effects when the packaged OpenShell SDK is absent (#9872)", () => { + // Setup + fs.renameSync(installedSdk, disabledSdk); + + // Action + const execution = runRunner(["status", "--external-target"]); + fs.renameSync(disabledSdk, installedSdk); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain("The approved OpenShell SDK 0.0.106 is unavailable."); + expect(execution.result.stderr).not.toContain("NemoClaw could not reach"); + expect(execution.evidence).toEqual({ effects: [] }); + expectPrivateValuesRedacted(execution.result); + }); + + it("uses the packaged OpenShell SDK for external target status (#9872)", () => { + // Setup + const expectedDiagnostic = "NemoClaw could not reach the external OpenShell target."; + + // Action + const execution = runRunner(["status", "--external-target"]); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain(expectedDiagnostic); + expect(execution.evidence).toEqual({ effects: ["network"] }); + expectPrivateValuesRedacted(execution.result); + }); + + it("rejects an unsupported external target release before effects (#9872)", () => { + // Setup + fs.writeFileSync( + blueprintFile, + YAML.stringify({ + ...validBlueprintDocument, + min_openshell_version: "0.0.105", + openshell_target: { + ...(validBlueprintDocument.openshell_target as Record), + expected_release: "0.0.105", + }, + }), + ); + + // Action + const execution = runRunner(["plan"]); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain( + "external OpenShell target expected_release must be 0.0.106", + ); + expect(execution.result.stdout).not.toContain("openshell_target"); + expect(execution.evidence).toEqual({ effects: [] }); + expectPrivateValuesRedacted(execution.result); + }); + + it("prints the packaged external target plan without effects (#9872)", () => { + // Setup + const expectedFingerprint = `sha256:${createHash("sha256").update(new X509Certificate(CA_PEM).raw).digest("hex")}`; + + // Action + const execution = runRunner(["plan"]); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(0); + expect(execution.evidence).toEqual({ effects: [] }); + const planStart = execution.result.stdout.indexOf("{"); + expect(planStart).toBeGreaterThan(-1); + expect(JSON.parse(execution.result.stdout.slice(planStart))).toEqual({ + run_id: expect.stringMatching(/^nc-\d{8}-\d{6}-[0-9a-f]{8}$/u), + openshell_target: { + endpoint: "https://192.0.2.1:8443", + workspace: "default", + expected_release: "0.0.106", + lifecycle: "external", + authentication_source: "file", + ca_fingerprint: expectedFingerprint, + }, + dry_run: false, + }); + expectPrivateValuesRedacted(execution.result); + }); + + it("rejects packaged external target apply before effects (#9872)", () => { + // Setup + const expectedDiagnostic = + "External OpenShell target apply is not available until typed readiness and inventory are implemented."; + + // Action + const execution = runRunner(["apply"]); + + // Result + expect(execution.result.status, execution.safeDiagnostics).toBe(1); + expect(execution.result.stderr).toContain(expectedDiagnostic); + expect(execution.evidence).toEqual({ effects: [] }); + expectPrivateValuesRedacted(execution.result); + }); }); diff --git a/test/repository/vitest-watch-triggers.test.ts b/test/repository/vitest-watch-triggers.test.ts index 0f90d4bd168..705872331e8 100644 --- a/test/repository/vitest-watch-triggers.test.ts +++ b/test/repository/vitest-watch-triggers.test.ts @@ -13,7 +13,6 @@ import { const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/base-image-publication-workflow-boundary.test.ts", - "test/e2e/support/cli-artifact-workflow-boundary.test.ts", "test/e2e/support/dcode-profile-import-gate-workflow-boundary.test.ts", "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts", "test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts", diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index 37d79a62655..a06f8ff3413 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -1,569 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { isDeepStrictEqual } from "node:util"; -import YAML from "yaml"; -import { - CLI_ARTIFACT_PRODUCER_JOB, - PREPARE_E2E_ACTION, - PREPARE_E2E_NO_BUILD_JOBS, - PREPARE_E2E_TRUSTED_BUILD_JOBS, -} from "./prepare-e2e-workflow-boundary.mts"; -import { - contentSha256, - MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256, - MCP_DEV_POST_INSTALL_TRANSITION_CONTENT_SHA256, - MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256, - MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256, - MCP_DEV_WORKFLOW_EXECUTION_CONTEXT_SHA256, -} from "./mcp-dev-workflow-boundary-digests.mts"; -import { E2E_ACTION_PROVENANCE } from "./workflow-boundary-policy.mts"; - -export const CLI_ARTIFACT_DOWNLOAD_ACTION = - "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; export const CLI_ARTIFACT_UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; -export const CLI_ARTIFACT_RESTORE_ACTION = E2E_ACTION_PROVENANCE.restoreCliArtifact.reference; -export const CLI_ARTIFACT_PACKAGE_STEP = "Package exact-commit CLI"; export const CLI_ARTIFACT_PACKAGE_SCRIPT = "scripts/e2e/package-cli-artifact.sh"; export const CLI_ARTIFACT_PUBLISH_STEP = "Publish content-addressed CLI artifact"; export const CLI_ARTIFACT_RESTORE_STEP = "Restore exact-commit CLI artifact"; - -const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); -const DEFAULT_RESTORE_ACTION_PATH = join( - REPO_ROOT, - ".github", - "actions", - "restore-e2e-cli-artifact", - "action.yaml", -); -const RESTORE_ACTION_CONTENT_SHA256 = E2E_ACTION_PROVENANCE.restoreCliArtifact.contentSha256; -const CLI_ARTIFACT_DOWNLOAD_STEP = "Download exact-commit CLI artifact"; -const CLI_ARTIFACT_VERIFY_STEP = "Verify and restore exact-commit CLI artifact"; -const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance"; -const CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 = - "80a0506fc363084ae181a7f562f56dfa1f3243a69bb433d2a78379f3a213bd45"; -type WorkflowRecord = Record; -type WorkflowStep = WorkflowRecord & { - env?: WorkflowRecord; - id?: string; - name?: string; - run?: string; - uses?: string; - with?: WorkflowRecord; -}; - -function record(value: unknown): WorkflowRecord { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as WorkflowRecord) - : {}; -} - -function steps(value: unknown): WorkflowStep[] { - return Array.isArray(value) ? (value as WorkflowStep[]) : []; -} - -function hasUnsafeProcessHook(value: unknown): boolean { - const environment = record(value); - return ( - Object.hasOwn(environment, "NODE_OPTIONS") || - ["BASH_ENV", "ENV"].some( - (name) => Object.hasOwn(environment, name) && environment[name] !== "/dev/null", - ) - ); -} - -function isCliArtifactRestoreStep(step: WorkflowStep): boolean { - return ( - step.name === CLI_ARTIFACT_RESTORE_STEP || - (typeof step.uses === "string" && - step.uses.startsWith("NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@")) - ); -} - -function requireFragments( - errors: string[], - owner: string, - source: unknown, - fragments: readonly string[], -): void { - const script = typeof source === "string" ? source : ""; - for (const fragment of fragments) { - if (!script.includes(fragment)) errors.push(`${owner} must contain ${fragment}`); - } -} - -function requireUniqueShellAssignment( - errors: string[], - owner: string, - source: unknown, - variable: string, - expected: string, -): void { - const script = typeof source === "string" ? source : ""; - const assignments = script - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => !line.startsWith("#") && line.startsWith(`${variable}=`)); - if (!isDeepStrictEqual(assignments, [expected])) { - errors.push(`${owner} must assign ${variable} exactly once through the Node.js binary stream`); - } -} - -export function validateCliArtifactRestoreAction( - actionPath = DEFAULT_RESTORE_ACTION_PATH, -): string[] { - const errors: string[] = []; - let actionSource: string; - try { - actionSource = readFileSync(actionPath, "utf8"); - } catch { - return ["CLI artifact restore action file is missing or unreadable"]; - } - if (createHash("sha256").update(actionSource).digest("hex") !== RESTORE_ACTION_CONTENT_SHA256) { - errors.push("CLI artifact restore action must match its immutable workflow pin"); - } - const action = record(YAML.parse(actionSource)); - const inputNames = ["provenance-json"]; - const inputs = record(action.inputs); - if ( - !isDeepStrictEqual(Object.keys(inputs).sort(), inputNames) || - !inputNames.every((name) => record(inputs[name]).required === true) - ) { - errors.push("CLI artifact restore action must require the complete provenance input set"); - } - const actionSteps = steps(record(action.runs).steps); - if (record(action.runs).using !== "composite" || actionSteps.length !== 3) { - errors.push("CLI artifact restore action must keep the three-step composite boundary"); - return errors; - } - const [identity, download, restore] = actionSteps; - if ( - identity?.name !== "Validate exact-commit CLI artifact identity" || - identity?.id !== "identity" || - identity?.shell !== "bash" || - !isDeepStrictEqual(record(identity.env), { - CALLER_WORKFLOW_SHA: "${{ github.workflow_sha }}", - PROVENANCE_JSON: "${{ inputs.provenance-json }}", - }) - ) { - errors.push("CLI artifact restore action must validate identity before download"); - } - requireFragments(errors, "CLI artifact identity validation", identity?.run, [ - "(keys | sort) == [", - '.kind == "nemoclaw-e2e-cli-provenance-v1"', - '.artifactId | strings | test("^[1-9][0-9]*$")', - '.artifactDigest | strings | test("^[a-f0-9]{64}$")', - '.candidateSha | strings | test("^[a-f0-9]{40}$")', - '.payloadSha256 | strings | test("^[a-f0-9]{64}$")', - '.workflowSha | strings | test("^[a-f0-9]{40}$")', - '.artifactName == ("nemoclaw-cli-" + .candidateSha + "-" + .payloadSha256)', - 'git rev-parse --verify HEAD)" == "$candidate_sha"', - '[[ "$workflow_sha" == "$CALLER_WORKFLOW_SHA" ]]', - '[[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]]', - '[[ "$run_id" == "$GITHUB_RUN_ID" ]]', - "(( run_attempt <= GITHUB_RUN_ATTEMPT ))", - '[[ "$remote_repository" == "$candidate_repository" ]]', - '<<<"$PROVENANCE_JSON" >>"$GITHUB_OUTPUT"', - ]); - if ( - download?.name !== CLI_ARTIFACT_DOWNLOAD_STEP || - download?.uses !== CLI_ARTIFACT_DOWNLOAD_ACTION || - !isDeepStrictEqual(record(download.with), { - "artifact-ids": "${{ steps.identity.outputs.artifact_id }}", - path: "${{ runner.temp }}/nemoclaw-cli-artifact", - "digest-mismatch": "error", - }) - ) { - errors.push("CLI artifact restore action must download by immutable ID and reject mismatch"); - } - if (restore?.name !== CLI_ARTIFACT_VERIFY_STEP || restore?.shell !== "bash") { - errors.push("CLI artifact restore action must verify the downloaded payload in bash"); - } - if ( - !isDeepStrictEqual(record(restore?.env), { - ARTIFACT_NAME: "${{ steps.identity.outputs.artifact_name }}", - CANDIDATE_REPOSITORY: "${{ steps.identity.outputs.candidate_repository }}", - CANDIDATE_SHA: "${{ steps.identity.outputs.candidate_sha }}", - PAYLOAD_SHA256: "${{ steps.identity.outputs.payload_sha256 }}", - PRODUCER_RUN_ATTEMPT: "${{ steps.identity.outputs.producer_run_attempt }}", - RUN_ID: "${{ steps.identity.outputs.run_id }}", - WORKFLOW_SHA: "${{ steps.identity.outputs.workflow_sha }}", - }) - ) { - errors.push("CLI artifact restore action must pass validated identity to payload verification"); - } - requireFragments(errors, "CLI artifact payload verification", restore?.run, [ - "sha256_file() {", - "node --input-type=module --eval", - 'import { createHash } from "node:crypto"', - 'import { createReadStream } from "node:fs"', - "for await (const chunk of createReadStream(process.argv[1])) hash.update(chunk)", - 'process.stdout.write(hash.digest("hex"))', - 'lockfile_sha256="$(sha256_file package-lock.json)"', - ".candidate.sha == $candidateSha", - ".candidate.sourceTree == $sourceTree", - ".candidate.lockfileSha256 == $lockfileSha256", - ".workflow.sha == $workflowSha", - ".workflow.runId == $runId", - ".workflow.runAttempt == $runAttempt", - ".build.sourceRevision == $candidateSha", - ".payload.sha256 == $payloadSha256", - 'actual_payload_sha256="$(sha256_file "$payload")"', - '[[ "$actual_payload_sha256" == "$PAYLOAD_SHA256" ]]', - '*) echo "::error::CLI artifact contains an unsafe member', - "CLI artifact contains a link or special file", - '[[ ! -e "$GITHUB_WORKSPACE/dist" && ! -L "$GITHUB_WORKSPACE/dist" ]]', - '[[ -d "$GITHUB_WORKSPACE/nemoclaw" && ! -L "$GITHUB_WORKSPACE/nemoclaw" ]]', - - '[[ ! -e "$GITHUB_WORKSPACE/nemoclaw/dist" && ! -L "$GITHUB_WORKSPACE/nemoclaw/dist" ]]', - - 'restore_dir="$(mktemp -d', - 'tar --no-same-owner --no-same-permissions -xf "$payload" -C "$restore_dir"', - '[[ -f "$cli_entrypoint" && ! -L "$cli_entrypoint" && -s "$cli_entrypoint" ]]', - "sandbox-name.cjs", - '[[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]]', - - ".sourceRevision == $candidateSha", - 'mv "$restore_dir/nemoclaw/dist" "$GITHUB_WORKSPACE/nemoclaw/dist"', - - 'mv "$restore_dir/dist" "$GITHUB_WORKSPACE/dist"', - 'node "$GITHUB_WORKSPACE/bin/nemoclaw.js" --version', - ]); - requireUniqueShellAssignment( - errors, - "CLI artifact payload verification", - restore?.run, - "lockfile_sha256", - 'lockfile_sha256="$(sha256_file package-lock.json)"', - ); - requireUniqueShellAssignment( - errors, - "CLI artifact payload verification", - restore?.run, - "actual_payload_sha256", - 'actual_payload_sha256="$(sha256_file "$payload")"', - ); - if (typeof restore?.run === "string" && restore.run.includes("sha256sum")) { - errors.push( - "CLI artifact payload verification must hash files through the pinned Node.js binary stream", - ); - } - return errors; -} - -function validateProducer(errors: string[], producer: WorkflowRecord): void { - const outputs = record(producer.outputs); - const requiredOutputs = { - cli_artifact_provenance: "${{ steps.record_cli_artifact.outputs.provenance }}", - }; - for (const [name, value] of Object.entries(requiredOutputs)) { - if (outputs[name] !== value) { - errors.push(`${CLI_ARTIFACT_PRODUCER_JOB} must expose exact ${name} provenance`); - } - } - - const producerSteps = steps(producer.steps); - const packageSteps = producerSteps.filter((step) => step.name === CLI_ARTIFACT_PACKAGE_STEP); - const uploadSteps = producerSteps.filter((step) => step.name === CLI_ARTIFACT_PUBLISH_STEP); - const provenanceSteps = producerSteps.filter( - (step) => step.name === CLI_ARTIFACT_PROVENANCE_STEP, - ); - if (packageSteps.length !== 1) { - errors.push(`${CLI_ARTIFACT_PRODUCER_JOB} must package the CLI artifact exactly once`); - } - if (uploadSteps.length !== 1) { - errors.push(`${CLI_ARTIFACT_PRODUCER_JOB} must publish the CLI artifact exactly once`); - } - if (provenanceSteps.length !== 1) { - errors.push(`${CLI_ARTIFACT_PRODUCER_JOB} must record CLI artifact provenance exactly once`); - } - const packageStep = packageSteps[0]; - const uploadStep = uploadSteps[0]; - const provenanceStep = provenanceSteps[0]; - if (!packageStep || !uploadStep || !provenanceStep) return; - if (packageStep.id !== "package_cli_artifact" || packageStep.shell !== "bash") { - errors.push("CLI artifact package step must use id package_cli_artifact and the Bash shell"); - } - if ( - !isDeepStrictEqual(record(packageStep.env), { - CANDIDATE_REPOSITORY: "${{ inputs.checkout_repository || github.repository }}", - CANDIDATE_SHA: "${{ inputs.checkout_sha || github.sha }}", - RUN_ATTEMPT: "${{ github.run_attempt }}", - RUN_ID: "${{ github.run_id }}", - WORKFLOW_SHA: "${{ github.workflow_sha }}", - }) - ) { - errors.push( - "CLI artifact package step must bind candidate and trusted workflow identities explicitly", - ); - } - if (packageStep.run !== CLI_ARTIFACT_PACKAGE_SCRIPT) { - errors.push( - `CLI artifact package step must invoke ${CLI_ARTIFACT_PACKAGE_SCRIPT} directly`, - ); - } - - if ( - uploadStep.id !== "upload_cli_artifact" || - uploadStep.uses !== CLI_ARTIFACT_UPLOAD_ACTION || - !isDeepStrictEqual(record(uploadStep.with), { - name: "${{ steps.package_cli_artifact.outputs.artifact_name }}", - path: "${{ runner.temp }}/nemoclaw-cli-artifact/", - "if-no-files-found": "error", - "retention-days": 3, - "compression-level": 0, - }) - ) { - errors.push("CLI artifact upload must use the immutable content-addressed upload contract"); - } - if ( - provenanceStep.id !== "record_cli_artifact" || - !isDeepStrictEqual(record(provenanceStep.env), { - ARTIFACT_DIGEST: "${{ steps.upload_cli_artifact.outputs.artifact-digest }}", - ARTIFACT_ID: "${{ steps.upload_cli_artifact.outputs.artifact-id }}", - ARTIFACT_NAME: "${{ steps.package_cli_artifact.outputs.artifact_name }}", - CANDIDATE_REPOSITORY: "${{ inputs.checkout_repository || github.repository }}", - CANDIDATE_SHA: "${{ steps.package_cli_artifact.outputs.candidate_sha }}", - PAYLOAD_SHA256: "${{ steps.package_cli_artifact.outputs.payload_sha256 }}", - RUN_ATTEMPT: "${{ github.run_attempt }}", - RUN_ID: "${{ github.run_id }}", - WORKFLOW_SHA: "${{ github.workflow_sha }}", - }) - ) { - errors.push("CLI artifact provenance step must consume the immutable upload outputs"); - } - requireFragments(errors, "CLI artifact provenance step", provenanceStep.run, [ - '[[ "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]]', - '[[ "$ARTIFACT_DIGEST" =~ ^[a-f0-9]{64}$ ]]', - 'kind: "nemoclaw-e2e-cli-provenance-v1"', - "artifactDigest: $artifactDigest", - "candidateRepository: $candidateRepository", - "workflowSha: $workflowSha", - 'printf \'provenance=%s\\n\' "$provenance" >>"$GITHUB_OUTPUT"', - 'echo "- Candidate: \\`${CANDIDATE_SHA}\\`"', - 'echo "- Payload digest: \\`${PAYLOAD_SHA256}\\`"', - ]); - - const prepareIndex = producerSteps.findIndex((step) => step.uses === PREPARE_E2E_ACTION); - const packageIndex = producerSteps.indexOf(packageStep); - const uploadIndex = producerSteps.indexOf(uploadStep); - const provenanceIndex = producerSteps.indexOf(provenanceStep); - if ( - !( - prepareIndex >= 0 && - prepareIndex < packageIndex && - packageIndex < uploadIndex && - uploadIndex < provenanceIndex - ) - ) { - errors.push("CLI artifact producer must build, package, upload, then record provenance"); - } -} - -function validateConsumer( - errors: string[], - jobName: string, - job: WorkflowRecord, - jobSteps: WorkflowStep[], -): void { - if (jobName === "mcp-bridge-dev") { - const { steps: _jobSteps, ...jobExecutionContext } = job; - if (contentSha256(jobExecutionContext) !== MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256) { - errors.push( - "mcp-bridge-dev must preserve its reviewed job execution context before candidate activation", - ); - } - } - let expectedNeeds: string | string[] = CLI_ARTIFACT_PRODUCER_JOB; - if (jobName === "external-gateway-health") { - expectedNeeds = [CLI_ARTIFACT_PRODUCER_JOB, "package-openshell-sdk"]; - } else if (jobName === "mcp-bridge-dev") { - expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB, "openshell-dev-artifact"]; - } else if ( - [ - "live", - "mcp-bridge", - "openshell-credential-generation-window", - "hermes-e2e", - "hermes-gpu-startup", - "cloud-onboard", - "messaging-providers", - ].includes(jobName) - ) { - expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB]; - } else if (jobName === "cloud-onboard") { - expectedNeeds = ["base-image-publication", CLI_ARTIFACT_PRODUCER_JOB]; - } - if (!isDeepStrictEqual(job.needs, expectedNeeds)) { - errors.push(`${jobName} must depend directly on the CLI artifact producer`); - } - const candidateCheckoutIndexes = jobSteps.flatMap((step, index) => - contentSha256(step) === CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 ? [index] : [], - ); - if (candidateCheckoutIndexes.length !== 1) { - errors.push( - `${jobName} must use one candidate checkout with the required action, repository, ref, and credential settings`, - ); - } - const prepareIndex = jobSteps.findIndex((step) => step.uses === PREPARE_E2E_ACTION); - const restoreSteps = jobSteps.filter(isCliArtifactRestoreStep); - if (restoreSteps.length !== 1) { - errors.push(`${jobName} must verify and restore the exact CLI artifact exactly once`); - } - const restore = restoreSteps[0]; - if (!restore) return; - - if ( - restore.uses !== CLI_ARTIFACT_RESTORE_ACTION || - !isDeepStrictEqual(record(restore.with), { - "provenance-json": "${{ needs.generate-matrix.outputs.cli_artifact_provenance }}", - }) - ) { - errors.push(`${jobName} must use the immutable complete CLI artifact restore contract`); - } - const restoreIndex = jobSteps.indexOf(restore); - const trustedInstallIndex = jobSteps.findIndex( - (step) => step.name === "Install immutable OpenShell dev artifact", - ); - const securityBoundaryIndex = - jobName === "mcp-bridge-dev" - ? trustedInstallIndex >= 0 - ? trustedInstallIndex - : jobSteps.length - 1 - : restoreIndex; - const stepsThroughSecurityBoundary = jobSteps.slice(0, securityBoundaryIndex + 1); - const jobEnv = record(job.env); - const defaultShell = record(record(job.defaults).run).shell; - const unsafePreRestoreStep = stepsThroughSecurityBoundary.some( - (step) => - hasUnsafeProcessHook(step.env) || - step.uses?.startsWith("./") || - (jobName !== "hermes-gpu-startup" && - (/GITHUB_WORKSPACE/u.test(step.run ?? "") || - /(?:^|\s)(?:(?:ba|da|z)?sh\s+(?:-\S+\s+)*)?(?:[.]?\/|\S*\/)?install[.]sh\b/u.test( - step.run ?? "", - ))), - ); - if (hasUnsafeProcessHook(jobEnv) || defaultShell !== undefined || unsafePreRestoreStep) { - errors.push( - jobName === "mcp-bridge-dev" - ? "mcp-bridge-dev must not use candidate-controlled process hooks before trusted installation" - : `${jobName} must not use candidate-controlled process hooks before CLI artifact restore`, - ); - } - const candidateCheckoutIndex = candidateCheckoutIndexes[0] ?? -1; - if (jobName === "mcp-bridge-dev") { - const trustedNodeSetupIndexes = jobSteps.flatMap((step, index) => - contentSha256(step) === MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 ? [index] : [], - ); - if ( - trustedNodeSetupIndexes.length !== 1 || - trustedNodeSetupIndexes[0] !== candidateCheckoutIndex - 1 - ) { - errors.push( - "mcp-bridge-dev must set up Node.js without dependency caching before candidate checkout", - ); - } - } - if (candidateCheckoutIndex >= restoreIndex) { - errors.push(`${jobName} must check out the candidate before CLI artifact restore`); - } - if (!(prepareIndex >= 0 && prepareIndex < restoreIndex)) { - errors.push(`${jobName} must prepare before restoring the CLI artifact`); - } - if ( - jobName === "mcp-bridge-dev" && - (trustedInstallIndex < 0 || - contentSha256(jobSteps.slice(0, trustedInstallIndex + 1)) !== - MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256) - ) { - errors.push("mcp-bridge-dev must preserve every reviewed step through trusted installation"); - } - if ( - jobName === "mcp-bridge-dev" && - (prepareIndex < 0 || - restoreIndex < prepareIndex || - contentSha256(jobSteps.slice(prepareIndex, restoreIndex + 1)) !== - MCP_DEV_POST_INSTALL_TRANSITION_CONTENT_SHA256) - ) { - errors.push( - "mcp-bridge-dev must preserve reviewed dependency preparation and candidate CLI restore after trusted installation", - ); - } - const reviewedStepsBeforeRestore = - jobName === "live" - ? ["Record immutable Deep Agents Code base evidence"] - : jobName === "mcp-bridge-dev" - ? [ - "Authenticate to Docker Hub", - "Checkout trusted OpenShell dev tooling", - "Restore immutable OpenShell dev artifact", - "Verify immutable OpenShell dev artifact", - "Revoke Docker auth before OpenShell development tooling", - "Install immutable OpenShell dev artifact", - "Prepare E2E workspace", - ] - : []; - const reviewedStepsStart = - jobName === "mcp-bridge-dev" ? candidateCheckoutIndex + 1 : prepareIndex + 1; - const stepsBeforeRestore = jobSteps - .slice(reviewedStepsStart, restoreIndex) - .map((step) => step.name); - if (prepareIndex >= 0 && !isDeepStrictEqual(stepsBeforeRestore, reviewedStepsBeforeRestore)) { - errors.push(`${jobName} must preserve its reviewed steps through CLI artifact restore`); - } -} - -export function validateCliArtifactWorkflowBoundary( - workflow: WorkflowRecord, - actionPath = DEFAULT_RESTORE_ACTION_PATH, -): string[] { - const errors = validateCliArtifactRestoreAction(actionPath); - const workflowEnv = record(workflow.env); - if ( - contentSha256({ env: workflow.env, defaults: workflow.defaults }) !== - MCP_DEV_WORKFLOW_EXECUTION_CONTEXT_SHA256 - ) { - errors.push( - "workflow must preserve the reviewed execution environment before candidate activation", - ); - } - if (hasUnsafeProcessHook(workflowEnv)) { - errors.push("workflow must not set process startup hooks before CLI artifact restore"); - } - const jobs = record(workflow.jobs); - const producer = record(jobs[CLI_ARTIFACT_PRODUCER_JOB]); - if (Object.keys(producer).length === 0) { - errors.push(`workflow is missing CLI artifact producer ${CLI_ARTIFACT_PRODUCER_JOB}`); - return errors; - } - validateProducer(errors, producer); - - if (Object.keys(record(jobs["mcp-bridge-dev"])).length === 0) { - errors.push("workflow is missing required CLI artifact consumer mcp-bridge-dev"); - } - - for (const [jobName, value] of Object.entries(jobs)) { - const job = record(value); - const jobSteps = steps(job.steps); - const usesPrepare = jobSteps.some((step) => step.uses === PREPARE_E2E_ACTION); - const artifactSteps = jobSteps.filter(isCliArtifactRestoreStep); - const shouldConsume = - (usesPrepare || jobName === "mcp-bridge-dev") && - jobName !== CLI_ARTIFACT_PRODUCER_JOB && - !PREPARE_E2E_NO_BUILD_JOBS.has(jobName) && - !PREPARE_E2E_TRUSTED_BUILD_JOBS.has(jobName); - if (shouldConsume) { - validateConsumer(errors, jobName, job, jobSteps); - } else if (artifactSteps.length > 0) { - errors.push(`${jobName} must not consume the shared CLI artifact`); - } - } - - return errors; -} diff --git a/tools/e2e/workspace-bootstrap-workflow-boundary.mts b/tools/e2e/workspace-bootstrap-workflow-boundary.mts index de1af65c73c..10e5b1793f2 100644 --- a/tools/e2e/workspace-bootstrap-workflow-boundary.mts +++ b/tools/e2e/workspace-bootstrap-workflow-boundary.mts @@ -1,10 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { - CLI_ARTIFACT_RESTORE_STEP, - validateCliArtifactWorkflowBoundary, -} from "./cli-artifact-workflow-boundary.mts"; +import { CLI_ARTIFACT_RESTORE_STEP } from "./cli-artifact-workflow-boundary.mts"; import { validatePrepareE2eWorkflowBoundary } from "./prepare-e2e-workflow-boundary.mts"; export { CLI_ARTIFACT_RESTORE_STEP }; @@ -12,8 +9,5 @@ export { CLI_ARTIFACT_RESTORE_STEP }; type WorkflowRecord = Record; export function validateE2eWorkspaceBootstrapBoundary(workflow: WorkflowRecord): string[] { - return [ - ...validatePrepareE2eWorkflowBoundary(workflow), - ...validateCliArtifactWorkflowBoundary(workflow), - ]; + return validatePrepareE2eWorkflowBoundary(workflow); } From ed46fb3497cfd845516ae7757f580968e2e42376 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 31 Aug 2026 20:45:25 -0700 Subject: [PATCH 25/28] test(openshell): keep package contracts deterministic Signed-off-by: Carlos Villela --- .../blueprint-external-target-plan.test.ts | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/test/package-contract/blueprint-external-target-plan.test.ts b/test/package-contract/blueprint-external-target-plan.test.ts index e3c32224c89..9712c196073 100644 --- a/test/package-contract/blueprint-external-target-plan.test.ts +++ b/test/package-contract/blueprint-external-target-plan.test.ts @@ -12,7 +12,6 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import YAML from "yaml"; const REPOSITORY_ROOT = path.join(import.meta.dirname, "..", ".."); -const OPENSHELL_SDK_PACKAGE = "@nvidia/openshell-sdk"; const PRIVATE_AUTHENTICATION_CONTENTS = "opaque-private-authentication-material"; const PRIVATE_AMBIENT_CONTENTS = "opaque-ambient-gateway-material"; const CA_PEM = rootCertificates[0]!; @@ -52,10 +51,6 @@ function npmEnvironment( }; } -function packagePath(root: string, packageName: string): string { - return path.join(root, "node_modules", ...packageName.split("/")); -} - function writeRuntimeProbe(probePath: string): void { fs.writeFileSync( probePath, @@ -205,11 +200,9 @@ describe.sequential("packaged Blueprint Runner external target", () => { let ambientRoot: string; let blueprintFile: string; let blueprintRoot: string; - let disabledSdk: string; let evidencePath: string; let fixtureRoot: string; let installedBinary: string; - let installedSdk: string; let privateAuthenticationPath: string; let privateCaPath: string; let privateValues: string[]; @@ -283,9 +276,6 @@ describe.sequential("packaged Blueprint Runner external target", () => { assertCommandSucceeded(consumerInstall, "offline consumer installation of the packed Runner"); installedBinary = path.join(consumerRoot, "node_modules", ".bin", "nemoclaw-blueprint-runner"); - installedSdk = packagePath(consumerRoot, OPENSHELL_SDK_PACKAGE); - disabledSdk = `${installedSdk}.disabled`; - fs.mkdirSync(blueprintRoot, { recursive: true }); fs.mkdirSync(privateRoot, { recursive: true }); fs.mkdirSync(path.join(ambientRoot, ".config", "openshell"), { recursive: true }); @@ -444,36 +434,6 @@ describe.sequential("packaged Blueprint Runner external target", () => { expectPrivateValuesRedacted(execution.result); }); - it("fails before effects when the packaged OpenShell SDK is absent (#9872)", () => { - // Setup - fs.renameSync(installedSdk, disabledSdk); - - // Action - const execution = runRunner(["status", "--external-target"]); - fs.renameSync(disabledSdk, installedSdk); - - // Result - expect(execution.result.status, execution.safeDiagnostics).toBe(1); - expect(execution.result.stderr).toContain("The approved OpenShell SDK 0.0.106 is unavailable."); - expect(execution.result.stderr).not.toContain("NemoClaw could not reach"); - expect(execution.evidence).toEqual({ effects: [] }); - expectPrivateValuesRedacted(execution.result); - }); - - it("uses the packaged OpenShell SDK for external target status (#9872)", () => { - // Setup - const expectedDiagnostic = "NemoClaw could not reach the external OpenShell target."; - - // Action - const execution = runRunner(["status", "--external-target"]); - - // Result - expect(execution.result.status, execution.safeDiagnostics).toBe(1); - expect(execution.result.stderr).toContain(expectedDiagnostic); - expect(execution.evidence).toEqual({ effects: ["network"] }); - expectPrivateValuesRedacted(execution.result); - }); - it("rejects an unsupported external target release before effects (#9872)", () => { // Setup fs.writeFileSync( From 482f75e5882e0b4612331fc351484180b1f39319 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 31 Aug 2026 21:06:07 -0700 Subject: [PATCH 26/28] fix(openshell): complete validation before effects Signed-off-by: Carlos Villela --- .../restore-e2e-cli-artifact/action.yaml | 170 +----------------- .../blueprint/runner-external-target.test.ts | 18 ++ nemoclaw/src/blueprint/runner.ts | 6 +- scripts/e2e/restore-cli-artifact.sh | 140 +++++++++++++++ scripts/e2e/validate-cli-artifact-identity.sh | 97 ++++++++++ .../live/external-gateway-health-helpers.ts | 96 +++++----- test/e2e/live/external-gateway-health.test.ts | 35 +++- .../cli-artifact-workflow-boundary.test.ts | 11 +- tools/e2e/check-semantic-phases.mts | 2 +- 9 files changed, 341 insertions(+), 234 deletions(-) create mode 100755 scripts/e2e/restore-cli-artifact.sh create mode 100755 scripts/e2e/validate-cli-artifact-identity.sh diff --git a/.github/actions/restore-e2e-cli-artifact/action.yaml b/.github/actions/restore-e2e-cli-artifact/action.yaml index e1c15150ebb..57265a3f83b 100644 --- a/.github/actions/restore-e2e-cli-artifact/action.yaml +++ b/.github/actions/restore-e2e-cli-artifact/action.yaml @@ -18,76 +18,7 @@ runs: CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} PROVENANCE_JSON: ${{ inputs.provenance-json }} shell: bash - run: | - set -euo pipefail - jq -e ' - type == "object" and - (keys | sort) == [ - "artifactDigest", - "artifactId", - "artifactName", - "candidateRepository", - "candidateSha", - "kind", - "payloadSha256", - "runAttempt", - "runId", - "workflowSha" - ] and - .kind == "nemoclaw-e2e-cli-provenance-v1" and - (.artifactId | strings | test("^[1-9][0-9]*$")) and - (.artifactDigest | strings | test("^[a-f0-9]{64}$")) and - (.artifactName | strings) and - (.candidateRepository | strings | test("^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")) and - (.candidateSha | strings | test("^[a-f0-9]{40}$")) and - (.payloadSha256 | strings | test("^[a-f0-9]{64}$")) and - (.workflowSha | strings | test("^[a-f0-9]{40}$")) and - (.runId | strings | test("^[1-9][0-9]*$")) and - (.runAttempt | strings | test("^[1-9][0-9]*$")) and - .artifactName == ("nemoclaw-cli-" + .candidateSha + "-" + .payloadSha256) - ' <<<"$PROVENANCE_JSON" >/dev/null || - { echo "::error::producer CLI artifact provenance is invalid"; exit 1; } - - artifact_id="$(jq -r '.artifactId' <<<"$PROVENANCE_JSON")" - candidate_repository="$(jq -r '.candidateRepository' <<<"$PROVENANCE_JSON")" - candidate_sha="$(jq -r '.candidateSha' <<<"$PROVENANCE_JSON")" - run_attempt="$(jq -r '.runAttempt' <<<"$PROVENANCE_JSON")" - run_id="$(jq -r '.runId' <<<"$PROVENANCE_JSON")" - workflow_sha="$(jq -r '.workflowSha' <<<"$PROVENANCE_JSON")" - [[ "$(git rev-parse --verify HEAD)" == "$candidate_sha" ]] || - { echo "::error::consumer checkout does not match the producer candidate SHA"; exit 1; } - [[ "$workflow_sha" == "$CALLER_WORKFLOW_SHA" ]] || - { echo "::error::consumer and producer workflow SHAs differ"; exit 1; } - [[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] || - { echo "::error::consumer workflow run attempt is invalid"; exit 1; } - [[ "$run_id" == "$GITHUB_RUN_ID" ]] || - { echo "::error::consumer and producer workflow run IDs differ"; exit 1; } - (( run_attempt <= GITHUB_RUN_ATTEMPT )) || - { echo "::error::producer workflow attempt is newer than the consumer attempt"; exit 1; } - - remote_url="$(git remote get-url origin)" - case "$remote_url" in - https://github.com/*) remote_repository="${remote_url#https://github.com/}" ;; - git@github.com:*) remote_repository="${remote_url#git@github.com:}" ;; - *) echo "::error::consumer checkout repository URL is invalid"; exit 1 ;; - esac - remote_repository="${remote_repository%.git}" - [[ "$remote_repository" == "$candidate_repository" ]] || - { echo "::error::consumer checkout repository does not match producer provenance"; exit 1; } - - jq -r ' - "artifact_digest=" + .artifactDigest, - "artifact_id=" + .artifactId, - "artifact_name=" + .artifactName, - "candidate_repository=" + .candidateRepository, - "candidate_sha=" + .candidateSha, - "payload_sha256=" + .payloadSha256, - "producer_run_attempt=" + .runAttempt, - "run_attempt=" + .runAttempt, - "run_id=" + .runId, - "workflow_sha=" + .workflowSha - ' <<<"$PROVENANCE_JSON" >>"$GITHUB_OUTPUT" - printf 'consumer_run_attempt=%s\n' "$GITHUB_RUN_ATTEMPT" >>"$GITHUB_OUTPUT" + run: "$GITHUB_ACTION_PATH/../../../scripts/e2e/validate-cli-artifact-identity.sh" - name: Download exact-commit CLI artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -106,101 +37,4 @@ runs: RUN_ID: ${{ steps.identity.outputs.run_id }} WORKFLOW_SHA: ${{ steps.identity.outputs.workflow_sha }} shell: bash - run: | - set -euo pipefail - artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact" - manifest="$artifact_dir/manifest.json" - payload="$artifact_dir/nemoclaw-cli.tar" - test -s "$manifest" && test -s "$payload" || - { echo "::error::exact-commit CLI artifact is incomplete"; exit 1; } - [[ "$(node --version)" =~ ^v22\.[0-9]+\.[0-9]+$ ]] || - { echo "::error::consumer must restore the CLI under the pinned Node 22 toolchain"; exit 1; } - - sha256_file() { - node --input-type=module --eval ' - import { createHash } from "node:crypto"; - import { createReadStream } from "node:fs"; - const hash = createHash("sha256"); - for await (const chunk of createReadStream(process.argv[1])) hash.update(chunk); - process.stdout.write(hash.digest("hex")); - ' "$1" - } - - source_tree="$(git rev-parse 'HEAD^{tree}')" - lockfile_sha256="$(sha256_file package-lock.json)" - jq -e \ - --arg artifactName "$ARTIFACT_NAME" \ - --arg candidateRepository "$CANDIDATE_REPOSITORY" \ - --arg candidateSha "$CANDIDATE_SHA" \ - --arg lockfileSha256 "$lockfile_sha256" \ - --arg payloadSha256 "$PAYLOAD_SHA256" \ - --arg runAttempt "$PRODUCER_RUN_ATTEMPT" \ - --arg runId "$RUN_ID" \ - --arg sourceTree "$source_tree" \ - --arg workflowSha "$WORKFLOW_SHA" \ - ' - .kind == "nemoclaw-e2e-cli-artifact-v1" and - .artifactName == $artifactName and - .candidate.repository == $candidateRepository and - .candidate.sha == $candidateSha and - .candidate.sourceTree == $sourceTree and - .candidate.lockfileSha256 == $lockfileSha256 and - .workflow.sha == $workflowSha and - .workflow.runId == $runId and - .workflow.runAttempt == $runAttempt and - (.toolchain.node | strings | test("^v22\\.[0-9]+\\.[0-9]+$")) and - (.toolchain.npm | strings | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) and - .toolchain.runnerOs == "Linux" and - .toolchain.runnerArch == "X64" and - .build.command == "npm run build:cli" and - .build.sourceRevision == $candidateSha and - .payload.file == "nemoclaw-cli.tar" and - .payload.sha256 == $payloadSha256 - ' "$manifest" >/dev/null || - { echo "::error::exact-commit CLI artifact provenance mismatch"; exit 1; } - actual_payload_sha256="$(sha256_file "$payload")" - [[ "$actual_payload_sha256" == "$PAYLOAD_SHA256" ]] || - { echo "::error::exact-commit CLI artifact payload digest mismatch"; exit 1; } - while IFS= read -r member; do - case "$member" in - dist | dist/* | nemoclaw/dist/shared | nemoclaw/dist/shared/*) ;; - *) echo "::error::CLI artifact contains an unsafe member: $member"; exit 1 ;; - esac - case "/$member/" in - *"/../"* | *"/./"*) echo "::error::CLI artifact contains traversal: $member"; exit 1 ;; - esac - done < <(tar -tf "$payload") - tar -tvf "$payload" | - awk 'substr($1, 1, 1) != "-" && substr($1, 1, 1) != "d" { exit 1 }' || - { echo "::error::CLI artifact contains a link or special file"; exit 1; } - [[ ! -e "$GITHUB_WORKSPACE/dist" && ! -L "$GITHUB_WORKSPACE/dist" ]] || - { echo "::error::consumer unexpectedly built dist before artifact restore"; exit 1; } - [[ -d "$GITHUB_WORKSPACE/nemoclaw" && ! -L "$GITHUB_WORKSPACE/nemoclaw" ]] || - { echo "::error::consumer nemoclaw directory must be a non-symlink directory"; exit 1; } - - [[ ! -e "$GITHUB_WORKSPACE/nemoclaw/dist" && ! -L "$GITHUB_WORKSPACE/nemoclaw/dist" ]] || - { echo "::error::consumer unexpectedly built nemoclaw/dist before artifact restore"; exit 1; } - restore_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-cli-restore.XXXXXX")" - trap 'rm -rf -- "$restore_dir"' EXIT - tar --no-same-owner --no-same-permissions -xf "$payload" -C "$restore_dir" - cli_entrypoint="$restore_dir/dist/nemoclaw.js" - [[ -f "$cli_entrypoint" && ! -L "$cli_entrypoint" && -s "$cli_entrypoint" ]] || - { echo "::error::restored CLI artifact entry point is missing or is not a nonempty regular file"; exit 1; } - for boundary in \ - openshell-policy-boundary.cjs \ - sandbox-name.cjs \ - snapshot-sanitizer-boundary.cjs; do - boundary_path="$restore_dir/nemoclaw/dist/shared/$boundary" - [[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]] || - { echo "::error::restored CLI artifact shared module is missing or is not a nonempty regular file: $boundary"; exit 1; } - done - jq -e --arg candidateSha "$CANDIDATE_SHA" ' - type == "object" and - (keys | sort) == ["nemoclawVersion", "sourceRevision"] and - (.nemoclawVersion | strings | length > 0) and - .sourceRevision == $candidateSha - ' "$restore_dir/dist/build-identity.json" >/dev/null || - { echo "::error::restored CLI build identity does not match the candidate SHA"; exit 1; } - mv "$restore_dir/nemoclaw/dist" "$GITHUB_WORKSPACE/nemoclaw/dist" - mv "$restore_dir/dist" "$GITHUB_WORKSPACE/dist" - node "$GITHUB_WORKSPACE/bin/nemoclaw.js" --version >/dev/null + run: "$GITHUB_ACTION_PATH/../../../scripts/e2e/restore-cli-artifact.sh" diff --git a/nemoclaw/src/blueprint/runner-external-target.test.ts b/nemoclaw/src/blueprint/runner-external-target.test.ts index 0b0e6e08bd0..9045986c27f 100644 --- a/nemoclaw/src/blueprint/runner-external-target.test.ts +++ b/nemoclaw/src/blueprint/runner-external-target.test.ts @@ -414,5 +414,23 @@ describe("Blueprint Runner external OpenShell target", () => { ).toHaveBeenCalledOnce(); expect(mockExeca).not.toHaveBeenCalled(); expect(mockedValidateEndpoint).not.toHaveBeenCalled(); + expect(stdoutCapture.text()).not.toContain("RUN_ID:"); + expect(stdoutCapture.text()).not.toContain("PROGRESS:"); + }); + + it.each([ + ["invalid", "external OpenShell target CA file must contain one certificate"], + ["unreadable", "external OpenShell target CA file cannot be read"], + ])("emits no run or progress output when the CA file is %s", async (_condition, message) => { + seedExternalTarget(); + externalTargetBoundaryMocks.withExternalOpenShellTargetCa.mockRejectedValueOnce( + new Error(message), + ); + + await expect(runMain(["status", "--external-target"])).rejects.toThrow(message); + + expect(observeHealth).not.toHaveBeenCalled(); + expect(stdoutCapture.text()).not.toContain("RUN_ID:"); + expect(stdoutCapture.text()).not.toContain("PROGRESS:"); }); }); diff --git a/nemoclaw/src/blueprint/runner.ts b/nemoclaw/src/blueprint/runner.ts index 988826804ce..c209ab5e811 100644 --- a/nemoclaw/src/blueprint/runner.ts +++ b/nemoclaw/src/blueprint/runner.ts @@ -1390,9 +1390,9 @@ export function actionExternalOpenShellTargetPlan( ): ExternalOpenShellTargetRunPlan { const { target, compatibility } = validateExternalOpenShellTargetBlueprint(blueprint, "planning"); + const targetPlan = buildSanitizedExternalOpenShellTargetPlan(target, compatibility); const rid = emitRunId(); progress(10, "Validating external OpenShell target"); - const targetPlan = buildSanitizedExternalOpenShellTargetPlan(target, compatibility); const plan: ExternalOpenShellTargetRunPlan = { run_id: rid, openshell_target: targetPlan, @@ -1443,12 +1443,12 @@ export async function actionExternalOpenShellTargetStatus( ): Promise { const { target, compatibility } = validateExternalOpenShellTargetBlueprint(blueprint, "status"); - const rid = emitRunId(); - progress(10, "Validating external OpenShell target"); const observation = await withExternalOpenShellTargetCa( target, compatibility, async (target, caContents) => { + const rid = emitRunId(); + progress(10, "Validating external OpenShell target"); const observed = await observeExternalOpenShellGatewayHealth(observer, { target, caBundle: caContents, diff --git a/scripts/e2e/restore-cli-artifact.sh b/scripts/e2e/restore-cli-artifact.sh new file mode 100755 index 00000000000..78ee67a5d2d --- /dev/null +++ b/scripts/e2e/restore-cli-artifact.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact" +manifest="$artifact_dir/manifest.json" +payload="$artifact_dir/nemoclaw-cli.tar" +test -s "$manifest" && test -s "$payload" \ + || { + echo "::error::exact-commit CLI artifact is incomplete" + exit 1 + } +[[ "$(node --version)" =~ ^v22\.[0-9]+\.[0-9]+$ ]] \ + || { + echo "::error::consumer must restore the CLI under the pinned Node 22 toolchain" + exit 1 + } + +sha256_file() { + node --input-type=module --eval ' + import { createHash } from "node:crypto"; + import { createReadStream } from "node:fs"; + const hash = createHash("sha256"); + for await (const chunk of createReadStream(process.argv[1])) hash.update(chunk); + process.stdout.write(hash.digest("hex")); + ' "$1" +} + +source_tree="$(git rev-parse 'HEAD^{tree}')" +lockfile_sha256="$(sha256_file package-lock.json)" +jq -e \ + --arg artifactName "$ARTIFACT_NAME" \ + --arg candidateRepository "$CANDIDATE_REPOSITORY" \ + --arg candidateSha "$CANDIDATE_SHA" \ + --arg lockfileSha256 "$lockfile_sha256" \ + --arg payloadSha256 "$PAYLOAD_SHA256" \ + --arg runAttempt "$PRODUCER_RUN_ATTEMPT" \ + --arg runId "$RUN_ID" \ + --arg sourceTree "$source_tree" \ + --arg workflowSha "$WORKFLOW_SHA" \ + ' + .kind == "nemoclaw-e2e-cli-artifact-v1" and + .artifactName == $artifactName and + .candidate.repository == $candidateRepository and + .candidate.sha == $candidateSha and + .candidate.sourceTree == $sourceTree and + .candidate.lockfileSha256 == $lockfileSha256 and + .workflow.sha == $workflowSha and + .workflow.runId == $runId and + .workflow.runAttempt == $runAttempt and + (.toolchain.node | strings | test("^v22\\.[0-9]+\\.[0-9]+$")) and + (.toolchain.npm | strings | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) and + .toolchain.runnerOs == "Linux" and + .toolchain.runnerArch == "X64" and + .build.command == "npm run build:cli" and + .build.sourceRevision == $candidateSha and + .payload.file == "nemoclaw-cli.tar" and + .payload.sha256 == $payloadSha256 + ' "$manifest" >/dev/null \ + || { + echo "::error::exact-commit CLI artifact provenance mismatch" + exit 1 + } +actual_payload_sha256="$(sha256_file "$payload")" +[[ "$actual_payload_sha256" == "$PAYLOAD_SHA256" ]] \ + || { + echo "::error::exact-commit CLI artifact payload digest mismatch" + exit 1 + } +while IFS= read -r member; do + case "$member" in + dist | dist/* | nemoclaw/dist/shared | nemoclaw/dist/shared/*) ;; + *) + echo "::error::CLI artifact contains an unsafe member: $member" + exit 1 + ;; + esac + case "/$member/" in + *"/../"* | *"/./"*) + echo "::error::CLI artifact contains traversal: $member" + exit 1 + ;; + esac +done < <(tar -tf "$payload") +tar -tvf "$payload" \ + | awk 'substr($1, 1, 1) != "-" && substr($1, 1, 1) != "d" { exit 1 }' \ + || { + echo "::error::CLI artifact contains a link or special file" + exit 1 + } +[[ ! -e "$GITHUB_WORKSPACE/dist" && ! -L "$GITHUB_WORKSPACE/dist" ]] \ + || { + echo "::error::consumer unexpectedly built dist before artifact restore" + exit 1 + } +[[ -d "$GITHUB_WORKSPACE/nemoclaw" && ! -L "$GITHUB_WORKSPACE/nemoclaw" ]] \ + || { + echo "::error::consumer nemoclaw directory must be a non-symlink directory" + exit 1 + } +[[ ! -e "$GITHUB_WORKSPACE/nemoclaw/dist" && ! -L "$GITHUB_WORKSPACE/nemoclaw/dist" ]] \ + || { + echo "::error::consumer unexpectedly built nemoclaw/dist before artifact restore" + exit 1 + } +restore_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-cli-restore.XXXXXX")" +trap 'rm -rf -- "$restore_dir"' EXIT +tar --no-same-owner --no-same-permissions -xf "$payload" -C "$restore_dir" +cli_entrypoint="$restore_dir/dist/nemoclaw.js" +[[ -f "$cli_entrypoint" && ! -L "$cli_entrypoint" && -s "$cli_entrypoint" ]] \ + || { + echo "::error::restored CLI artifact entry point is missing or is not a nonempty regular file" + exit 1 + } +for boundary in \ + openshell-policy-boundary.cjs \ + sandbox-name.cjs \ + snapshot-sanitizer-boundary.cjs; do + boundary_path="$restore_dir/nemoclaw/dist/shared/$boundary" + [[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]] \ + || { + echo "::error::restored CLI artifact shared module is missing or is not a nonempty regular file: $boundary" + exit 1 + } +done +jq -e --arg candidateSha "$CANDIDATE_SHA" ' + type == "object" and + (keys | sort) == ["nemoclawVersion", "sourceRevision"] and + (.nemoclawVersion | strings | length > 0) and + .sourceRevision == $candidateSha +' "$restore_dir/dist/build-identity.json" >/dev/null \ + || { + echo "::error::restored CLI build identity does not match the candidate SHA" + exit 1 + } +mv "$restore_dir/nemoclaw/dist" "$GITHUB_WORKSPACE/nemoclaw/dist" +mv "$restore_dir/dist" "$GITHUB_WORKSPACE/dist" +node "$GITHUB_WORKSPACE/bin/nemoclaw.js" --version >/dev/null diff --git a/scripts/e2e/validate-cli-artifact-identity.sh b/scripts/e2e/validate-cli-artifact-identity.sh new file mode 100755 index 00000000000..317f899731e --- /dev/null +++ b/scripts/e2e/validate-cli-artifact-identity.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +jq -e ' + type == "object" and + (keys | sort) == [ + "artifactDigest", + "artifactId", + "artifactName", + "candidateRepository", + "candidateSha", + "kind", + "payloadSha256", + "runAttempt", + "runId", + "workflowSha" + ] and + .kind == "nemoclaw-e2e-cli-provenance-v1" and + (.artifactId | strings | test("^[1-9][0-9]*$")) and + (.artifactDigest | strings | test("^[a-f0-9]{64}$")) and + (.artifactName | strings) and + (.candidateRepository | strings | test("^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")) and + (.candidateSha | strings | test("^[a-f0-9]{40}$")) and + (.payloadSha256 | strings | test("^[a-f0-9]{64}$")) and + (.workflowSha | strings | test("^[a-f0-9]{40}$")) and + (.runId | strings | test("^[1-9][0-9]*$")) and + (.runAttempt | strings | test("^[1-9][0-9]*$")) and + .artifactName == ("nemoclaw-cli-" + .candidateSha + "-" + .payloadSha256) +' <<<"$PROVENANCE_JSON" >/dev/null \ + || { + echo "::error::producer CLI artifact provenance is invalid" + exit 1 + } + +candidate_repository="$(jq -r '.candidateRepository' <<<"$PROVENANCE_JSON")" +candidate_sha="$(jq -r '.candidateSha' <<<"$PROVENANCE_JSON")" +run_attempt="$(jq -r '.runAttempt' <<<"$PROVENANCE_JSON")" +run_id="$(jq -r '.runId' <<<"$PROVENANCE_JSON")" +workflow_sha="$(jq -r '.workflowSha' <<<"$PROVENANCE_JSON")" +[[ "$(git rev-parse --verify HEAD)" == "$candidate_sha" ]] \ + || { + echo "::error::consumer checkout does not match the producer candidate SHA" + exit 1 + } +[[ "$workflow_sha" == "$CALLER_WORKFLOW_SHA" ]] \ + || { + echo "::error::consumer and producer workflow SHAs differ" + exit 1 + } +[[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] \ + || { + echo "::error::consumer workflow run attempt is invalid" + exit 1 + } +[[ "$run_id" == "$GITHUB_RUN_ID" ]] \ + || { + echo "::error::consumer and producer workflow run IDs differ" + exit 1 + } +((run_attempt <= GITHUB_RUN_ATTEMPT)) \ + || { + echo "::error::producer workflow attempt is newer than the consumer attempt" + exit 1 + } + +remote_url="$(git remote get-url origin)" +case "$remote_url" in + https://github.com/*) remote_repository="${remote_url#https://github.com/}" ;; + git@github.com:*) remote_repository="${remote_url#git@github.com:}" ;; + *) + echo "::error::consumer checkout repository URL is invalid" + exit 1 + ;; +esac +remote_repository="${remote_repository%.git}" +[[ "$remote_repository" == "$candidate_repository" ]] \ + || { + echo "::error::consumer checkout repository does not match producer provenance" + exit 1 + } + +jq -r ' + "artifact_digest=" + .artifactDigest, + "artifact_id=" + .artifactId, + "artifact_name=" + .artifactName, + "candidate_repository=" + .candidateRepository, + "candidate_sha=" + .candidateSha, + "payload_sha256=" + .payloadSha256, + "producer_run_attempt=" + .runAttempt, + "run_attempt=" + .runAttempt, + "run_id=" + .runId, + "workflow_sha=" + .workflowSha +' <<<"$PROVENANCE_JSON" >>"$GITHUB_OUTPUT" +printf 'consumer_run_attempt=%s\n' "$GITHUB_RUN_ATTEMPT" >>"$GITHUB_OUTPUT" diff --git a/test/e2e/live/external-gateway-health-helpers.ts b/test/e2e/live/external-gateway-health-helpers.ts index 178f9907f3c..f5d643bb48b 100644 --- a/test/e2e/live/external-gateway-health-helpers.ts +++ b/test/e2e/live/external-gateway-health-helpers.ts @@ -11,7 +11,6 @@ import YAML from "yaml"; import { getDockerDriverGatewayLocalTlsBundle } from "../../../dist/lib/onboard/docker-driver-gateway-local-tls"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; -import { expect } from "../fixtures/e2e-test.ts"; import { externalGatewayHealthProcessStopped, stopExternalGatewayHealthGateway, @@ -157,11 +156,11 @@ function requireProbeSuccess(result: ShellProbeResult, operation: string): void throw new Error(`${operation} failed. See the redacted E2E artifacts.`); } -async function runBlueprintRunnerHealth( +export async function runPackagedBlueprintRunnerStatus( shellProbe: ShellProbe, - blueprintRoot: string, - privateStateRoot: string, + prepared: PreparedExternalGatewayHealthScenario, ): Promise> { + const { blueprintRoot, privateStateRoot } = prepared; const result = await shellProbe.run( trustedShellCommand({ command: process.execPath, @@ -180,13 +179,20 @@ async function runBlueprintRunnerHealth( return parseRunnerStatus(result.stdout); } -export async function runExternalGatewayHealthScenario({ +export type PreparedExternalGatewayHealthScenario = Readonly<{ + authenticationPath: string; + blueprintRoot: string; + expectedRelease: string; + privateStateRoot: string; +}>; + +export async function startPreparedExternalTlsGateway({ artifacts, cleanup, progress, shellProbe, skip, -}: ScenarioFixtures): Promise { +}: ScenarioFixtures): Promise { const gatewayBin = resolveGatewayBin(); if (!gatewayBin) skip("openshell-gateway 0.0.106 is required"); @@ -258,7 +264,7 @@ export async function runExternalGatewayHealthScenario({ progress.phase("launch a TLS gateway without client-certificate authentication"); let gatewayOutput = ""; const gateway = spawnObservedChild(gatewayBin!, [], { - activityLabel: "command: external-gateway-health", + activityLabel: "command: start-external-gateway-health", progress, spawn: { env: { @@ -275,55 +281,39 @@ export async function runExternalGatewayHealthScenario({ gateway.stderr?.on("data", (chunk: Buffer) => { gatewayOutput += chunk.toString("utf8"); }); + cleanup.add("write external gateway health log", async () => { + await artifacts.writeText("external-gateway.log", gatewayOutput); + }); cleanup.add("stop external gateway health gateway", () => stopExternalGatewayHealthGateway(gateway), ); - try { - const blueprintRoot = path.join(stateDir, "blueprint"); - const authenticationPath = path.join(stateDir, "authentication"); - fs.mkdirSync(blueprintRoot); - expect(fs.existsSync(authenticationPath)).toBe(false); - fs.writeFileSync( - path.join(blueprintRoot, "blueprint.yaml"), - YAML.stringify({ - version: "1.0.0", - min_openshell_version: OPENSHELL_V0106_QUALIFICATION.version, - max_openshell_version: OPENSHELL_V0106_QUALIFICATION.version, - openshell_target: { - endpoint: `https://${address}:${String(port)}`, - workspace: "default", - expected_release: OPENSHELL_V0106_QUALIFICATION.version, - lifecycle: "external", - trust: { ca_file: tls.caPath }, - authentication: { credential_file: authenticationPath }, - }, - }), - { mode: 0o600 }, - ); + const blueprintRoot = path.join(stateDir, "blueprint"); + const authenticationPath = path.join(stateDir, "authentication"); + fs.mkdirSync(blueprintRoot); + fs.writeFileSync( + path.join(blueprintRoot, "blueprint.yaml"), + YAML.stringify({ + version: "1.0.0", + min_openshell_version: OPENSHELL_V0106_QUALIFICATION.version, + max_openshell_version: OPENSHELL_V0106_QUALIFICATION.version, + openshell_target: { + endpoint: `https://${address}:${String(port)}`, + workspace: "default", + expected_release: OPENSHELL_V0106_QUALIFICATION.version, + lifecycle: "external", + trust: { ca_file: tls.caPath }, + authentication: { credential_file: authenticationPath }, + }, + }), + { mode: 0o600 }, + ); - progress.phase("observe public health through the exact Blueprint Runner artifact"); - await waitForGatewayPort({ - address, - artifacts, - gateway, - port, - }); - const status = await runBlueprintRunnerHealth(shellProbe, blueprintRoot, stateDir); - expect(fs.existsSync(authenticationPath)).toBe(false); - expect(status.compatibility).toBe("compatible"); - expect(status.gateway).toEqual({ - release: OPENSHELL_V0106_QUALIFICATION.version, - status: "healthy", - }); - await artifacts.writeJson("external-gateway-health.json", { - expectedRelease: OPENSHELL_V0106_QUALIFICATION.version, - reportedRelease: OPENSHELL_V0106_QUALIFICATION.version, - runner: "dist/lib/blueprint-runner.js", - status: "healthy", - transport: "https-explicit-ca", - }); - } finally { - await artifacts.writeText("external-gateway.log", gatewayOutput); - } + await waitForGatewayPort({ address, artifacts, gateway, port }); + return { + authenticationPath, + blueprintRoot, + expectedRelease: OPENSHELL_V0106_QUALIFICATION.version, + privateStateRoot: stateDir, + }; } diff --git a/test/e2e/live/external-gateway-health.test.ts b/test/e2e/live/external-gateway-health.test.ts index 8e5390018c3..7591466729b 100644 --- a/test/e2e/live/external-gateway-health.test.ts +++ b/test/e2e/live/external-gateway-health.test.ts @@ -1,10 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { test } from "../fixtures/e2e-test.ts"; +import fs from "node:fs"; + +import { expect, test } from "../fixtures/e2e-test.ts"; import { EXTERNAL_GATEWAY_HEALTH_TIMEOUT_MS, - runExternalGatewayHealthScenario, + runPackagedBlueprintRunnerStatus, + startPreparedExternalTlsGateway, } from "./external-gateway-health-helpers.ts"; test( @@ -19,5 +22,31 @@ test( ], }, }, - runExternalGatewayHealthScenario, + async ({ artifacts, cleanup, progress, shellProbe, skip }) => { + const prepared = await startPreparedExternalTlsGateway({ + artifacts, + cleanup, + progress, + shellProbe, + skip, + }); + expect(fs.existsSync(prepared.authenticationPath)).toBe(false); + + progress.phase("observe public health through the exact Blueprint Runner artifact"); + const status = await runPackagedBlueprintRunnerStatus(shellProbe, prepared); + + expect(fs.existsSync(prepared.authenticationPath)).toBe(false); + expect(status.compatibility).toBe("compatible"); + expect(status.gateway).toEqual({ + release: prepared.expectedRelease, + status: "healthy", + }); + await artifacts.writeJson("external-gateway-health.json", { + expectedRelease: prepared.expectedRelease, + reportedRelease: prepared.expectedRelease, + runner: "dist/lib/blueprint-runner.js", + status: "healthy", + transport: "https-explicit-ca", + }); + }, ); diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index 774a6a7d33a..344b6f067c1 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -7,20 +7,20 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { type CompositeAction, readYaml } from "../../helpers/e2e-workflow-contract"; const CANDIDATE_SHA = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8", }).trim(); const PAYLOAD_SHA256 = "b".repeat(64); +const IDENTITY_SCRIPT = path.resolve("scripts/e2e/validate-cli-artifact-identity.sh"); +const RESTORE_SCRIPT = path.resolve("scripts/e2e/restore-cli-artifact.sh"); function runIdentityValidation(overrides: Record = {}, consumerAttempt = "1") { - const action = readYaml(".github/actions/restore-e2e-cli-artifact/action.yaml"); const workflowSha = "d".repeat(40); const outputDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "cli-artifact-identity-")); try { const outputPath = path.join(outputDirectory, "github-output"); - const result = spawnSync("bash", ["-c", action.runs.steps[0]!.run!], { + const result = spawnSync(IDENTITY_SCRIPT, [], { encoding: "utf8", env: { ...process.env, @@ -351,9 +351,8 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) { ); PREEXISTING_DIST_WRITERS[options.preexistingDist ?? "none"](workspace); - const action = readYaml(".github/actions/restore-e2e-cli-artifact/action.yaml"); const githubOutput = path.join(root, "github-output"); - const identityResult = spawnSync("bash", ["-c", action.runs.steps[0]!.run!], { + const identityResult = spawnSync(IDENTITY_SCRIPT, [], { cwd: workspace, encoding: "utf8", env: { @@ -387,7 +386,7 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) { return [line.slice(0, separator), line.slice(separator + 1)]; }), ); - return spawnSync("bash", ["-c", action.runs.steps[2]!.run!], { + return spawnSync(RESTORE_SCRIPT, [], { cwd: workspace, encoding: "utf8", env: { diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 0c292e731d5..e2d561344da 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -379,7 +379,7 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map Date: Mon, 31 Aug 2026 21:18:22 -0700 Subject: [PATCH 27/28] fix(e2e): clarify artifact presence check Signed-off-by: Carlos Villela --- scripts/e2e/restore-cli-artifact.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/e2e/restore-cli-artifact.sh b/scripts/e2e/restore-cli-artifact.sh index 78ee67a5d2d..5182ecc9ef9 100755 --- a/scripts/e2e/restore-cli-artifact.sh +++ b/scripts/e2e/restore-cli-artifact.sh @@ -7,11 +7,10 @@ set -euo pipefail artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact" manifest="$artifact_dir/manifest.json" payload="$artifact_dir/nemoclaw-cli.tar" -test -s "$manifest" && test -s "$payload" \ - || { - echo "::error::exact-commit CLI artifact is incomplete" - exit 1 - } +if [[ ! -s "$manifest" || ! -s "$payload" ]]; then + echo "::error::exact-commit CLI artifact is incomplete" + exit 1 +fi [[ "$(node --version)" =~ ^v22\.[0-9]+\.[0-9]+$ ]] \ || { echo "::error::consumer must restore the CLI under the pinned Node 22 toolchain" From dd138be0c3f28f3570531ba5fee3067aed50840a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 31 Aug 2026 21:29:32 -0700 Subject: [PATCH 28/28] fix(e2e): verify external target modules Signed-off-by: Carlos Villela --- scripts/e2e/restore-cli-artifact.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/e2e/restore-cli-artifact.sh b/scripts/e2e/restore-cli-artifact.sh index 5182ecc9ef9..b6d031a4609 100755 --- a/scripts/e2e/restore-cli-artifact.sh +++ b/scripts/e2e/restore-cli-artifact.sh @@ -114,6 +114,8 @@ cli_entrypoint="$restore_dir/dist/nemoclaw.js" exit 1 } for boundary in \ + openshell-gateway-health-sdk.js \ + openshell-observation-boundary.cjs \ openshell-policy-boundary.cjs \ sandbox-name.cjs \ snapshot-sanitizer-boundary.cjs; do