diff --git a/.github/workflows/brev-nightly-e2e.yaml b/.github/workflows/brev-nightly-e2e.yaml index fe2df1a810..0574c3bbe9 100644 --- a/.github/workflows/brev-nightly-e2e.yaml +++ b/.github/workflows/brev-nightly-e2e.yaml @@ -7,7 +7,7 @@ name: E2E / Brev Nightly # main E2E workflow remains isolated from Brev platform flakiness. # # Suites: -# all credential-sanitization + telegram-injection +# telegram-injection Telegram command-injection regression coverage # messaging-providers Telegram + Discord provider/L7 proxy validation # messaging-compatible-endpoint local compatible-endpoint Telegram validation # full install/onboard/inference/CLI path @@ -43,7 +43,7 @@ jobs: strategy: fail-fast: false matrix: - test_suite: [all, messaging-providers, messaging-compatible-endpoint, full] + test_suite: [telegram-injection, messaging-providers, messaging-compatible-endpoint, full] uses: ./.github/workflows/e2e-branch-validation.yaml with: # Bind tested code to the ref that supplied this reviewed workflow. Do diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index 1490464a06..ec5d74c6a8 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -19,7 +19,7 @@ name: E2E / Branch Validation # WHEN TO USE: # - Before merging a PR that touches onboard, sandbox, security, or infra # - To validate a branch works end-to-end on a clean Linux environment -# - To run security regression suites (credential-sanitization, injection) +# - To run the Telegram command-injection regression suite # - Manually via workflow_dispatch, or called from other workflows # # NOTE: This does NOT test a published Brev Launchable image. It bootstraps a @@ -29,10 +29,6 @@ name: E2E / Branch Validation # full — Install → onboard → sandbox verify → live inference # against NVIDIA Endpoints → CLI operations. Tests the # complete user journey. (~10 min, destroys sandbox) -# credential-sanitization — 24 tests validating PR #743: credential stripping from -# migration snapshots, auth-profiles.json deletion, blueprint -# digest verification, symlink traversal protection, and -# runtime sandbox credential checks. Requires running sandbox. # telegram-injection — 18 tests validating PR #584: command injection prevention # through $(cmd), backticks, quote breakout, ${VAR} expansion, # process table leak checks, and SANDBOX_NAME validation. @@ -47,8 +43,6 @@ name: E2E / Branch Validation # dashboard-remote-bind — Verifies opt-in remote dashboard forwards bind 0.0.0.0. # gpu — Provisions a Brev GPU VM and runs the Ollama GPU E2E # sandbox proof suite from source. (~45 min) -# all — Runs credential-sanitization + telegram-injection, each with -# its own sandbox lifecycle (NOT the independent full journey). # # Required secrets: BREV_API_KEY + BREV_ORG_ID (or legacy BREV_API_TOKEN), NVIDIA_INFERENCE_API_KEY # Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance) @@ -67,13 +61,11 @@ on: type: choice options: - full - - credential-sanitization - telegram-injection - messaging-providers - messaging-compatible-endpoint - dashboard-remote-bind - gpu - - all keep_alive: description: "Keep Brev instance alive after tests (for SSH debugging)" required: false @@ -179,7 +171,7 @@ jobs: TEST_SUITE: ${{ inputs.test_suite }} run: | case "$TEST_SUITE" in - full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all) ;; + full|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu) ;; *) echo "::error::test_suite is not one of the supported Brev E2E suites" exit 1 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 90c4c6eceb..440b99f44c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -346,7 +346,7 @@ jobs: retired-selector-compatibility: needs: generate-matrix - if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }} + if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -1982,103 +1982,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - credential-sanitization: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.targets), ',credential-sanitization,') }} - runs-on: ubuntu-latest - timeout-minutes: 65 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "credential-sanitization" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/credential-sanitization - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-credential-sanitization" - OPENSHELL_GATEWAY: "nemoclaw" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - *dockerhub-auth - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - - - name: Run credential sanitization live test - # Preserves the - # same ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY lane by running - # install.sh, onboarding a real sandbox, and probing sandbox state from - # Vitest while fixture redaction owns evidence logs. - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - run: | - set -euo pipefail - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/credential-sanitization.test.ts - - - name: Upload credential sanitization artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - - - name: Clean up Docker auth - if: always() - shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh - - credential-migration: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.targets), ',credential-migration,') }} - runs-on: ubuntu-latest - timeout-minutes: 50 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "credential-migration" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/credential-migration - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-cred-migration" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - *dockerhub-auth - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - - - name: Run credential migration live test - # This live test - # stages NVIDIA_INFERENCE_API_KEY through legacy credentials.json as the - # custom provider's COMPATIBLE_API_KEY. The hosted service behind this - # repo-scoped secret is inference-api.nvidia.com, not Build/NVIDIA - # Endpoints, so the test must exercise the compatible-provider route. - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - NEMOCLAW_PROVIDER: custom - NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 - NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-ultra - NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-ultra - NEMOCLAW_PREFERRED_API: openai-completions - run: | - set -euo pipefail - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/credential-migration.test.ts - - - name: Upload credential migration artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - - - name: Clean up Docker auth - if: always() - shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh - sessions-agents-cli: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',sessions-agents-cli,') || contains(format(',{0},', inputs.targets), ',sessions-agents-cli,') }} @@ -4724,54 +4627,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - diagnostics: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.targets), ',diagnostics,') }} - runs-on: ubuntu-latest - timeout-minutes: 60 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "diagnostics" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/diagnostics - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-diag" - OPENSHELL_GATEWAY: "nemoclaw" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - *dockerhub-auth - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - - - name: Run diagnostics live test - # This preserves the - # ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY lane by running - # debug archives, install.sh/onboard, sandbox exec/status, and - # gateway-backed credentials list/reset from Vitest. - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - run: | - set -euo pipefail - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/diagnostics.test.ts - - - name: Upload diagnostics artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - - - name: Clean up Docker auth - if: always() - shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh - # Scheduled release-baseline coverage for #6108. This runs independently of # the current lifecycle lane so the exact v0.0.71 build does not extend that # lane's critical path. @@ -5883,8 +5738,6 @@ jobs: brave-search, ollama-auth-proxy, cron-preflight-inference-local, - credential-sanitization, - credential-migration, sessions-agents-cli, hermes-e2e, hermes-gpu-startup, @@ -5920,7 +5773,6 @@ jobs: model-router-provider-routed-inference, sandbox-operations, sandbox-survival, - diagnostics, snapshot-commands, openclaw-plugin-runtime-exdev-release, openclaw-plugin-runtime-exdev, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index cec7308408..0aff8ff064 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -246,11 +246,6 @@ "test": "rejects credential-backed provider smokes in the PR-safe inference-routing job", "category": "security" }, - { - "file": "test/e2e/support/e2e-workflow.test.ts", - "test": "rejects diagnostics workflow-boundary drift for secret and Docker auth handling", - "category": "security" - }, { "file": "test/e2e/support/e2e-workflow.test.ts", "test": "rejects duplicate unguarded Docker Hub auth in messaging-compatible-endpoint", diff --git a/src/lib/security/credential-filter-secret-patterns.test.ts b/src/lib/security/credential-filter-secret-patterns.test.ts index bc948665e3..8e22ffda16 100644 --- a/src/lib/security/credential-filter-secret-patterns.test.ts +++ b/src/lib/security/credential-filter-secret-patterns.test.ts @@ -26,8 +26,10 @@ describe("isCredentialField", () => { expect(isCredentialField("clientSecret")).toBe(true); expect(isCredentialField("bearerToken")).toBe(true); expect(isCredentialField("privateKey")).toBe(true); + expect(isCredentialField("signingKey")).toBe(true); expect(isCredentialField("sessionToken")).toBe(true); expect(isCredentialField("sessionKey")).toBe(true); + expect(isCredentialField("authKey")).toBe(true); // OpenClaw channel token fields (#5027). expect(isCredentialField("botToken")).toBe(true); expect(isCredentialField("appToken")).toBe(true); @@ -103,6 +105,10 @@ describe("isCredentialField", () => { expect(isCredentialField("PATH")).toBe(false); expect(isCredentialField("tokenizer")).toBe(false); expect(isCredentialField("maxTokens")).toBe(false); + expect(isCredentialField("displayName")).toBe(false); + expect(isCredentialField("sortKey")).toBe(false); + expect(isCredentialField("sessionId")).toBe(false); + expect(isCredentialField("accessLevel")).toBe(false); expect(isCredentialField("X-Request-Id")).toBe(false); expect(isCredentialField("author")).toBe(false); }); diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index 797b4cc85d..0d2b19f3ff 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -271,7 +271,7 @@ describe("Brev nightly workflow contract", () => { expect(validation?.env?.TEST_SUITE).toBe("${{ inputs.test_suite }}"); expect(validation?.run).toContain( - "full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all", + "full|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu", ); expect(validation?.run).toContain("exit 1"); expect(steps.indexOf(validation as NonNullable)).toBeLessThan( diff --git a/test/brev-remote-vitest.test.ts b/test/brev-remote-vitest.test.ts index 988429d8a6..613cbfc9ba 100644 --- a/test/brev-remote-vitest.test.ts +++ b/test/brev-remote-vitest.test.ts @@ -20,7 +20,7 @@ import { buildBrevRemoteVitestCommand, } from "../tools/e2e/brev-remote-vitest.mts"; -const TARGET = "test/e2e/live/credential-sanitization.test.ts"; +const TARGET = "test/e2e/live/telegram-injection.test.ts"; type Fixture = { fakeBin: string; @@ -109,18 +109,15 @@ describe("Brev remote Vitest command", () => { }); it("does not seed shared harness state for suites that own their sandbox lifecycle", () => { - expect(brevSuiteNeedsHarnessSandbox("all")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("full")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("gpu")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("messaging-compatible-endpoint")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("messaging-providers")).toBe(false); - expect(brevSuiteHarnessSandboxName("all")).toBeUndefined(); expect(brevSuiteHarnessSandboxName("messaging-compatible-endpoint")).toBeUndefined(); expect(brevSuiteHarnessSandboxName("messaging-providers")).toBeUndefined(); }); it("preserves harness onboarding for single-target suites", () => { - expect(brevSuiteNeedsHarnessSandbox("credential-sanitization")).toBe(true); expect(brevSuiteNeedsHarnessSandbox("telegram-injection")).toBe(true); expect(brevSuiteNeedsHarnessSandbox("dashboard-remote-bind")).toBe(true); expect(brevSuiteHarnessSandboxName("dashboard-remote-bind")).toBe("e2e-test"); diff --git a/test/credential-migration-reconciliation.test.ts b/test/credential-migration-reconciliation.test.ts new file mode 100644 index 0000000000..23289ce4c6 --- /dev/null +++ b/test/credential-migration-reconciliation.test.ts @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; +import { + removeLegacyCredentialsFile, + stageLegacyCredentialsToEnv, +} from "../src/lib/credentials/store.js"; +import { + type CredentialProviderRegistrationDeps, + createCredentialProviderRegistration, +} from "../src/lib/onboard/credential-provider-registration.js"; +import { handleFinalizationState } from "../src/lib/onboard/machine/handlers/finalization.js"; +import type { MessagingTokenDef } from "../src/lib/onboard/messaging-prep.js"; +import type { Session } from "../src/lib/state/onboard-session.js"; +import { withProcessEnv } from "./support/setup-inference-test-harness.js"; + +const LEGACY_SECRET = "sk-TEST-NOT-A-REAL-STORED-KEY"; + +type RegistrationAttempt = ReturnType< + ReturnType["stageSandboxCredentialProviders"] +>; + +const REGISTRATION_SCENARIOS = [ + { + label: "keeps plaintext after gateway registration fails", + registrationStatus: 1, + settle: async (attempt: RegistrationAttempt) => { + await expect(attempt).rejects.toThrow("gateway registration exited 1"); + }, + expectedFilePresent: true, + expectedMigrated: false, + }, + { + label: "removes plaintext after gateway registration succeeds", + registrationStatus: 0, + settle: async (attempt: RegistrationAttempt) => { + await expect(attempt).resolves.toEqual([ + { name: "legacy-openai", type: "generic", credentialEnv: "OPENAI_API_KEY" }, + ]); + }, + expectedFilePresent: false, + expectedMigrated: true, + }, +] as const; + +async function finalizeMigration( + stagedLegacyKeys: readonly string[], + migratedLegacyKeys: ReadonlySet, +): Promise { + await handleFinalizationState({ + sandboxName: "test-box", + model: "gpt-5.4", + provider: "openai-api", + nimContainer: null, + agent: { + runtime: { kind: "terminal", interactive_command: "test-agent" }, + }, + hermesAuthMethod: null, + hermesToolGateways: [], + stagedLegacyKeys, + migratedLegacyKeys, + webSearchEnabled: false, + deps: { + ensureAgentDashboardForward: () => 0, + setDefaultSandbox: () => undefined, + recordPostVerifyStarted: async () => ({}) as Session, + toSessionUpdates: (updates) => updates, + removeLegacyCredentialsFile, + cleanupStaleHostFiles: () => undefined, + checkAndRecoverSandboxProcesses: () => undefined, + warmupScopeUpgrade: () => undefined, + autoPairScopeApproval: () => undefined, + getChatUiUrl: () => "", + buildVerifyChain: () => null, + verifyDeployment: async () => null, + formatVerificationDiagnostics: () => [], + isDeploymentHealthy: () => true, + reportDeploymentReadiness: () => undefined, + verifyWebSearchInsideSandbox: () => undefined, + printDashboard: () => undefined, + error: () => undefined, + log: () => undefined, + }, + }); +} + +describe("legacy credential reconciliation", () => { + it.each(REGISTRATION_SCENARIOS)("$label (#7617)", async (scenario) => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-credential-migration-")); + const legacyDir = path.join(tmpDir, ".nemoclaw"); + const legacyFile = path.join(legacyDir, "credentials.json"); + fs.mkdirSync(legacyDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + legacyFile, + JSON.stringify({ + OPENAI_API_KEY: LEGACY_SECRET, + OPENSHELL_GATEWAY: "tampered-gateway", + NODE_OPTIONS: "--require=/tmp/tampered.js", + }), + { mode: 0o600 }, + ); + const exit = vi.spyOn(process, "exit").mockImplementation((code) => { + throw new Error(`gateway registration exited ${String(code)}`); + }); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + await withProcessEnv( + { + HOME: tmpDir, + OPENAI_API_KEY: undefined, + OPENSHELL_GATEWAY: "trusted-gateway", + NODE_OPTIONS: "--enable-source-maps", + }, + async () => { + const stagedLegacyKeys = stageLegacyCredentialsToEnv(); + const stagedLegacyValues = new Map( + stagedLegacyKeys.map((key) => [key, process.env[key] ?? ""]), + ); + const migratedLegacyKeys = new Set(); + const session = { stagedCredentialProviders: [] } as unknown as Session; + const runOpenshell = vi.fn((args: string[], _options?: unknown) => ({ + status: args.slice(0, 2).join(" ") === "provider get" ? 1 : scenario.registrationStatus, + stdout: "", + stderr: scenario.registrationStatus === 0 ? "" : "registration failed", + })); + const deps: CredentialProviderRegistrationDeps = { + root: path.join(import.meta.dirname, ".."), + runOpenshell: + runOpenshell as unknown as CredentialProviderRegistrationDeps["runOpenshell"], + redact: (input) => input, + getGatewayName: () => "nemoclaw", + normalizeCredentialValue: (value) => (typeof value === "string" ? value.trim() : ""), + updateSession: (mutator) => mutator(session) ?? session, + stagedLegacyValues, + migratedLegacyKeys, + persistMigratedLegacyKeys: () => undefined, + }; + const registration = createCredentialProviderRegistration(deps); + const tokenDefs: MessagingTokenDef[] = [ + { + name: "legacy-openai", + envKey: "OPENAI_API_KEY", + token: process.env.OPENAI_API_KEY ?? "", + }, + ]; + + await scenario.settle( + registration.stageSandboxCredentialProviders( + { + sandboxName: "test-box", + enabledChannels: [], + webSearchConfig: null, + agent: {}, + }, + async () => ({ messagingTokenDefs: tokenDefs }), + ), + ); + + await finalizeMigration(stagedLegacyKeys, migratedLegacyKeys); + + expect(stagedLegacyKeys).toEqual(["OPENAI_API_KEY"]); + expect(process.env.OPENAI_API_KEY).toBe(LEGACY_SECRET); + expect(process.env.OPENSHELL_GATEWAY).toBe("trusted-gateway"); + expect(process.env.NODE_OPTIONS).toBe("--enable-source-maps"); + expect(migratedLegacyKeys.has("OPENAI_API_KEY")).toBe(scenario.expectedMigrated); + expect(runOpenshell.mock.calls.flatMap(([args]) => args)).not.toContain(LEGACY_SECRET); + expect(runOpenshell.mock.calls.find(([args]) => args[1] === "create")?.[1]).toMatchObject( + { + env: { OPENAI_API_KEY: LEGACY_SECRET }, + }, + ); + expect( + JSON.stringify(runOpenshell.mock.calls), + "tampered non-credential fields must not reach gateway registration", + ).not.toMatch(/tampered-gateway|tampered\.js/); + expect(exit).toHaveBeenCalledTimes(scenario.registrationStatus); + expect( + fs.existsSync(legacyFile), + scenario.expectedFilePresent + ? "failed registration must preserve the legacy file" + : "successful registration must remove the legacy file", + ).toBe(scenario.expectedFilePresent); + }, + ); + } finally { + error.mockRestore(); + exit.mockRestore(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e-recommendations.test.ts b/test/e2e-recommendations.test.ts index c28b7d7faa..ad02fc75f7 100644 --- a/test/e2e-recommendations.test.ts +++ b/test/e2e-recommendations.test.ts @@ -47,11 +47,7 @@ const ADVERSARIAL_E2E_TEXT = [ "nice gh secret list", "command aws secretsmanager get-secret-value --secret-id prod", ]; -const E2E_CONTROL_PLANE_JOB_IDS = new Set([ - "cloud-onboard", - "credential-sanitization", - "security-posture", -]); +const E2E_CONTROL_PLANE_JOB_IDS = new Set(["cloud-onboard", "cloud-inference", "security-posture"]); function withoutControlPlaneRecommendations( recommendations: readonly T[], @@ -242,7 +238,7 @@ describe("E2E recommendation normalizer", () => { { domain: "runtime", reason: command, confidence: "high", matchedFiles: [] }, ], requiredTests: [{ id: "security-posture", reason: command }], - optionalTests: [{ id: "credential-sanitization", reason: command }], + optionalTests: [{ id: "cloud-inference", reason: command }], newE2eRecommendations: [ { domain: "runtime", reason: "Add coverage.", suggestedTest: command, priority: "high" }, ], @@ -626,8 +622,8 @@ describe("E2E recommendation normalizer", () => { ); expect(normalized.required.map((item) => item.id)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", ]); expect(normalized.optional).toEqual([]); @@ -681,8 +677,8 @@ describe("E2E recommendation normalizer", () => { ); expect(normalized.required.map((item) => item.id)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", ]); expect(normalized.required.map((item) => item.id)).not.toContain("string-only"); @@ -733,8 +729,8 @@ describe("E2E recommendation normalizer", () => { ); expect(normalized.required.map((item) => item.id)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", ]); expect(normalized.required.map((item) => item.id)).not.toContain("retired-proof"); @@ -788,8 +784,8 @@ describe("E2E recommendation normalizer", () => { ); expect(normalized.required.map((item) => item.id)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", "full-e2e", "hermes-e2e", @@ -851,8 +847,8 @@ jobs: ); expect(normalized.required.map((item) => [item.selectorType, item.id])).toEqual([ + ["job", "cloud-inference"], ["job", "cloud-onboard"], - ["job", "credential-sanitization"], ["job", "security-posture"], ["job", "token-rotation"], ]); @@ -890,8 +886,8 @@ jobs: ); expect(normalized.required.map((item) => [item.selectorType, item.id])).toEqual([ + ["job", "cloud-inference"], ["job", "cloud-onboard"], - ["job", "credential-sanitization"], ["job", "security-posture"], ["job", "token-rotation"], ]); @@ -926,8 +922,8 @@ jobs: ); expect(normalized.required.map((item) => item.id)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", ]); expect(normalized.required.map((item) => item.id)).not.toContain("steal-secrets"); @@ -941,9 +937,9 @@ jobs: { e2eWorkflowText: String.raw` jobs: - cloud-inference: + inference-routing: steps: - # inputs.jobs ,cloud-inference, + # inputs.jobs ,inference-routing, # test/e2e/live/comment-only.test.ts - run: echo harmless `, @@ -951,11 +947,11 @@ jobs: ); expect(normalized.required.map((item) => item.id)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", ]); - expect(normalized.required.map((item) => item.id)).not.toContain("cloud-inference"); + expect(normalized.required.map((item) => item.id)).not.toContain("inference-routing"); }); it("removes optional recommendations whose id duplicates a required one", () => { diff --git a/test/e2e/README.md b/test/e2e/README.md index fe6d1f66e3..df82ef5825 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -554,7 +554,7 @@ The closed-PR outcome also applies when a fork repository was deleted and GitHub consequently returns no head-repository object. Shared sandbox-boundary changes have a floor of `full-e2e`, `hermes-e2e`, and `security-posture`. E2E control-plane changes select `cloud-onboard`, -`credential-sanitization`, and `security-posture`. The `e2e-control-plane` +`cloud-inference`, and `security-posture`. The `e2e-control-plane` family is a conservative path boundary that includes non-documentation files under `tools/e2e/` and `test/e2e/`, plus the E2E and PR-CI workflows, risk policy, dependency and test configuration, and preparation and upload actions. diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index bf165ea2a4..594c319a3f 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -26,8 +26,8 @@ * * Optional env vars: * TEST_SUITE — which test to run: full (default), deploy-cli, gpu, - * credential-sanitization, telegram-injection, messaging-providers, - * messaging-compatible-endpoint, dashboard-remote-bind, all + * telegram-injection, messaging-providers, + * messaging-compatible-endpoint, dashboard-remote-bind * BREV_MIN_VCPU — Minimum vCPUs for CPU instance (default: 4) * BREV_MIN_RAM — Minimum RAM in GB for CPU instance (default: 16) * BREV_PROVIDER — Cloud provider filter for brev search (default: gcp for CPU, any for GPU) @@ -1213,8 +1213,7 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { }, 120_000); // 2 min for cleanup // NOTE: The full E2E test runs install.sh --non-interactive and owns the - // complete sandbox lifecycle. The composite security suite also lets each - // remote target own that lifecycle, without a shared harness registry. + // complete sandbox lifecycle. it.runIf(TEST_SUITE === "full")( "full E2E suite passes on remote VM", () => { @@ -1233,20 +1232,7 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { 1_800_000, ); - it.runIf(TEST_SUITE === "credential-sanitization" || TEST_SUITE === "all")( - "credential sanitization suite passes on remote VM", - () => { - const output = runRemoteVitest( - "e2e-live", - "test/e2e/live/credential-sanitization.test.ts", - BREV_SECURITY_SUITE_TIMEOUT_MS, - ); - expectVitestPassed(output); - }, - BREV_SECURITY_SUITE_TIMEOUT_MS + BREV_REMOTE_WRAPPER_GRACE_MS, - ); - - it.runIf(TEST_SUITE === "telegram-injection" || TEST_SUITE === "all")( + it.runIf(TEST_SUITE === "telegram-injection")( "telegram bridge injection suite passes on remote VM", () => { const output = runRemoteVitest( diff --git a/test/e2e/live/cloud-inference.test.ts b/test/e2e/live/cloud-inference.test.ts index 03aab21719..6ca30032ae 100644 --- a/test/e2e/live/cloud-inference.test.ts +++ b/test/e2e/live/cloud-inference.test.ts @@ -57,6 +57,7 @@ const CLOUD_MODEL = "nvidia/nemotron-3-super-120b-a12b"; const INSTALL_TIMEOUT_MS = 25 * 60_000; const CHAT_TIMEOUT_MS = 120_000; +const SANDBOX_PROBE_TIMEOUT_MS = 120_000; const TEST_TIMEOUT_MS = 40 * 60_000; const MAX_ATTEMPTS = positiveInteger(process.env.E2E_PHASE_5B_MAX_ATTEMPTS, 3); const RETRY_SLEEP_MS = positiveInteger(process.env.E2E_PHASE_5B_RETRY_SLEEP_SEC, 5) * 1_000; @@ -225,6 +226,86 @@ async function expectLiveChatPong( throw new Error(`Live chat failed after ${MAX_ATTEMPTS} attempt(s): ${lastFailure}`); } +async function expectSandboxCredentialBoundary( + sandbox: SandboxClient, + home: string, + apiKey: string, +): Promise { + const authProbe = await sandbox.exec( + SANDBOX_NAME, + [ + "sh", + "-lc", + "find /sandbox -name auth-profiles.json -not -path '*/node_modules/*' -not -path '*/dist/*' -print", + ], + { + artifactName: "phase-3-sandbox-auth-profiles-probe", + env: testEnv(home), + timeoutMs: SANDBOX_PROBE_TIMEOUT_MS, + }, + ); + expect(authProbe.exitCode, resultText(authProbe)).toBe(0); + expect(authProbe.stdout.trim(), "auth-profiles.json must not be present in sandbox state").toBe( + "", + ); + + const secretScanCommand = [ + "for dir in /sandbox/.openclaw /sandbox/.nemoclaw; do", + ' [ -d "$dir" ] || continue', + ` matches=$(grep -rIlE 'nvapi-|ghp_|npm_' "$dir")`, + " scan_status=$?", + ' case "$scan_status" in', + ` 0) filtered=$(printf '%s\\n' "$matches" | grep -Ev '/policies/|/plugin-runtime-deps/|/extensions/[^/]+/(dist|node_modules)/')`, + " filter_status=$?", + ' case "$filter_status" in', + " 0) filtered_file=$(mktemp)", + " temp_status=$?", + ' case "$temp_status" in 0) ;; *) exit "$temp_status" ;; esac', + ` trap 'rm -f "$filtered_file"' EXIT HUP INT TERM`, + ` printf '%s\\n' "$filtered" > "$filtered_file"`, + " write_status=$?", + ' case "$write_status" in 0) ;; *) exit "$write_status" ;; esac', + " while IFS= read -r file; do", + ` matching_lines=$(grep -IE 'nvapi-|ghp_|npm_' "$file")`, + " match_status=$?", + ' case "$match_status" in', + ` 0) printf '%s' "$matching_lines" | grep -qv 'STRIPPED'`, + " unstripped_status=$?", + ' case "$unstripped_status" in', + ` 0) printf '%s\\n' "$file" ;;`, + " 1) ;;", + ' *) exit "$unstripped_status" ;;', + " esac", + " ;;", + " 1) ;;", + ' *) exit "$match_status" ;;', + " esac", + ' done < "$filtered_file"', + ' rm -f "$filtered_file"', + " trap - EXIT HUP INT TERM", + " ;;", + " 1) ;;", + ' *) exit "$filter_status" ;;', + " esac", + " ;;", + " 1) ;;", + ' *) exit "$scan_status" ;;', + " esac", + "done", + ].join("\n"); + + const secretProbe = await sandbox.exec(SANDBOX_NAME, ["sh", "-lc", secretScanCommand], { + artifactName: "phase-3-sandbox-secret-pattern-probe", + env: testEnv(home), + redactionValues: [apiKey], + timeoutMs: SANDBOX_PROBE_TIMEOUT_MS, + }); + expect(secretProbe.exitCode, resultText(secretProbe)).toBe(0); + expect(secretProbe.stdout.trim(), "sandbox config must not contain secret-shaped tokens").toBe( + "", + ); +} + // biome-ignore format: preserve legacy live-test body formatting so phase-only changes stay reviewable. test( "cloud inference: inference.local chat and OpenClaw skill filesystem validate", @@ -235,6 +316,7 @@ test( "verify cloud inference prerequisites", "install hosted-inference OpenClaw sandbox", "exercise managed inference.local chat", + "scan sandbox agent state for credentials", "validate repo and sandbox skill layouts", ], }, @@ -262,6 +344,7 @@ test( "install.sh --non-interactive creates or recreates the named OpenClaw sandbox", "nemoclaw and openshell are available on PATH after install", "curl inside the sandbox reaches https://inference.local/v1/chat/completions and returns PONG", + "sandbox agent state contains neither auth-profiles.json nor secret-shaped credential values", "repo .agents/skills SKILL.md frontmatter and body validate", "sandbox /sandbox/.openclaw and openclaw.json validate; skills subdir may be present or absent", ], @@ -337,9 +420,12 @@ test( content: chat.content, }); + progress.phase("scan sandbox agent state for credentials"); + await expectSandboxCredentialBoundary(sandbox, home, apiKey); + progress.phase("validate repo and sandbox skill layouts"); const repoSkills = await host.command("bash", [REPO_SKILL_VALIDATOR, "--repo", REPO_ROOT], { - artifactName: "phase-3-validate-repo-skills", + artifactName: "phase-4-validate-repo-skills", cwd: REPO_ROOT, env: testEnv(home), timeoutMs: 60_000, @@ -347,7 +433,7 @@ test( expect(repoSkills.exitCode, resultText(repoSkills)).toBe(0); const sandboxSkills = await host.command("bash", [SANDBOX_SKILL_VALIDATOR], { - artifactName: "phase-3-validate-sandbox-openclaw-skills", + artifactName: "phase-4-validate-sandbox-openclaw-skills", cwd: REPO_ROOT, env: testEnv(home, { SANDBOX_NAME }), timeoutMs: 90_000, @@ -367,6 +453,7 @@ test( dockerRunning: docker.exitCode === 0, installCompleted: install.exitCode === 0, chatReturnedPong: /pong/i.test(chat.content), + sandboxCredentialBoundaryValidated: true, repoSkillsValidated: repoSkills.exitCode === 0, sandboxOpenClawLayoutValidated: sandboxSkills.exitCode === 0, sandboxSkillsSubdir: sandboxSkillStatus, diff --git a/test/e2e/live/cloud-onboard.test.ts b/test/e2e/live/cloud-onboard.test.ts index 1abaa6ccc3..495cb30b12 100644 --- a/test/e2e/live/cloud-onboard.test.ts +++ b/test/e2e/live/cloud-onboard.test.ts @@ -25,9 +25,10 @@ const LIVE_TIMEOUT_MS = 60 * 60_000; validateSandboxName(SANDBOX_NAME); function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const home = extra.HOME || os.homedir(); return { ...buildAvailabilityProbeEnv(), - PATH: `${os.homedir()}/.local/bin:${os.homedir()}/.npm-global/bin:${process.env.PATH ?? ""}`, + PATH: `${home}/.local/bin:${home}/.npm-global/bin:${os.homedir()}/.local/bin:${os.homedir()}/.npm-global/bin:${process.env.PATH ?? ""}`, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_RECREATE_SANDBOX: "1", @@ -42,13 +43,13 @@ function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { async function cleanup( host: HostCliClient, sandbox: SandboxClient, - options: { verify: boolean; label: string }, + options: { verify: boolean; label: string; home: string }, ): Promise { const args = [path.join(REPO_ROOT, "test/e2e/e2e-cloud-experimental/cleanup.sh")]; if (options.verify) args.push("--verify"); const cleanupResult = await host.command("bash", args, { artifactName: `${options.label}-cloud-experimental-cleanup`, - env: env(), + env: env({ HOME: options.home }), timeoutMs: 180_000, }); if (options.verify) { @@ -57,7 +58,7 @@ async function cleanup( const gatewayDestroy = await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: `${options.label}-openshell-gateway-destroy`, - env: env(), + env: env({ HOME: options.home }), timeoutMs: 60_000, }); if (options.verify && gatewayDestroy.exitCode !== 0) { @@ -76,8 +77,11 @@ test("cloud onboard: public installer creates healthy sandbox with security chec meta: { e2ePhases: [ "check cloud onboarding prerequisites", + "stage legacy plaintext credential", "install and onboard cloud sandbox", + "verify migrated gateway credential", "validate installed CLI and corporate CA trust", + "collect scoped diagnostics from onboarded sandbox", "run cloud inference and security checks", "remove cloud sandbox", ], @@ -89,6 +93,15 @@ test("cloud onboard: public installer creates healthy sandbox with security chec process.env.NEMOCLAW_INSTALL_SCRIPT_URL ?? `https://raw.githubusercontent.com/NVIDIA/NemoClaw/${ref}/install.sh`; const installCwd = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-public-install-")); + const testHome = path.join(installCwd, "home"); + const legacyDir = path.join(testHome, ".nemoclaw"); + const legacyFile = path.join(legacyDir, "credentials.json"); + const testEnv = (extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv => + env({ HOME: testHome, ...extra }); + const hostedEnvWithoutCredentials = { ...hosted.env }; + delete hostedEnvWithoutCredentials[hosted.sourceSecretName]; + delete hostedEnvWithoutCredentials[hosted.credentialEnv]; + fs.mkdirSync(testHome, { recursive: true, mode: 0o700 }); const corporateCa = createCorporateCaFixture("explicit", "nemoclaw-cloud-corporate-ca-"); cleanupRegistry.trackDisposable("remove public installer workspace", () => fs.rmSync(installCwd, { recursive: true, force: true }), @@ -107,8 +120,12 @@ test("cloud onboard: public installer creates healthy sandbox with security chec corporateCaSource: corporateCa.sourceLabel, contracts: [ "public curl installer uses GitHub clone path for the requested ref", + "ordinary cloud onboard migrates an allowlisted legacy credential through the real gateway", + "tampered non-credential legacy fields do not become gateway providers", + "successful onboard removes plaintext credentials.json", "sandbox appears healthy after cloud onboarding", "explicit corporate CA source is baked and merged with OpenShell trust inside the sandbox", + "installed CLI creates a non-empty diagnostics archive for the registered sandbox", "cloud split checks cover inference.local, security leak checks, and Landlock/read-only behavior", "cleanup verifies sandbox removal", ], @@ -116,7 +133,7 @@ test("cloud onboard: public installer creates healthy sandbox with security chec const docker = await host.command("docker", ["info"], { artifactName: "phase-0-docker-info", - env: env(), + env: testEnv(), timeoutMs: 30_000, }); if (docker.exitCode !== 0) { @@ -125,9 +142,25 @@ test("cloud onboard: public installer creates healthy sandbox with security chec } cleanupRegistry.trackDisposable("remove cloud-onboard sandbox", () => - cleanup(host, sandbox, { label: "cleanup", verify: true }), + cleanup(host, sandbox, { home: testHome, label: "cleanup", verify: true }), + ); + await cleanup(host, sandbox, { home: testHome, label: "pre-cleanup", verify: false }); + + progress.phase("stage legacy plaintext credential"); + fs.mkdirSync(legacyDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + legacyFile, + JSON.stringify( + { + [hosted.credentialEnv]: hosted.apiKey, + OPENSHELL_GATEWAY: "evil-gw-from-tampered-file", + NODE_OPTIONS: "--require=/tmp/evil.js", + }, + null, + 2, + ), + { mode: 0o600 }, ); - await cleanup(host, sandbox, { label: "pre-cleanup", verify: false }); progress.phase("install and onboard cloud sandbox"); const install = await host.command( @@ -135,10 +168,9 @@ test("cloud onboard: public installer creates healthy sandbox with security chec ["-lc", `cd ${shellQuote(installCwd)} && curl -fsSL ${shellQuote(installUrl)} | bash`], { artifactName: "phase-1-public-install", - env: env({ - ...hosted.env, + env: testEnv({ + ...hostedEnvWithoutCredentials, ...corporateCa.env, - NVIDIA_INFERENCE_API_KEY: hosted.apiKey, NEMOCLAW_INSTALL_REF: ref, NEMOCLAW_INSTALL_TAG: ref, NEMOCLAW_INSTALL_SCRIPT_URL: installUrl, @@ -150,8 +182,33 @@ test("cloud onboard: public installer creates healthy sandbox with security chec expect(install.exitCode, resultText(install)).toBe(0); expect(resultText(install)).toContain("Installing NemoClaw from GitHub"); expect(resultText(install)).toContain("Cloning NemoClaw source"); + expect(resultText(install)).toContain( + "Staged 1 legacy credential(s) for migration to the OpenShell gateway.", + ); if (ref !== "main") expect(resultText(install)).toContain(`Resolved install ref: ${ref}`); + progress.phase("verify migrated gateway credential"); + expect(fs.existsSync(legacyFile), "successful onboard must remove legacy credentials.json").toBe( + false, + ); + const providers = await host.command( + "openshell", + ["-g", "nemoclaw", "provider", "list", "--names"], + { + artifactName: "phase-2-gateway-provider-list", + env: testEnv(), + timeoutMs: 60_000, + }, + ); + expect(providers.exitCode, resultText(providers)).toBe(0); + const providerNames = providers.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /^[a-zA-Z][a-zA-Z0-9_-]*$/.test(line)); + expect(providerNames).toContain(hosted.providerName); + expect(providerNames).not.toContain("OPENSHELL_GATEWAY"); + expect(providerNames).not.toContain("NODE_OPTIONS"); + progress.phase("validate installed CLI and corporate CA trust"); const cliProbe = await host.command( "bash", @@ -159,13 +216,13 @@ test("cloud onboard: public installer creates healthy sandbox with security chec "-lc", 'export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"; command -v nemoclaw; command -v openshell; nemoclaw --help >/dev/null', ], - { artifactName: "phase-2-cli-path-probe", env: env(), timeoutMs: 60_000 }, + { artifactName: "phase-2-cli-path-probe", env: testEnv(), timeoutMs: 60_000 }, ); expect(cliProbe.exitCode, resultText(cliProbe)).toBe(0); const list = await host.command("bash", ["-lc", "nemoclaw list"], { artifactName: "phase-2-nemoclaw-list", - env: env(), + env: testEnv(), timeoutMs: 60_000, }); expect(list.exitCode, resultText(list)).toBe(0); @@ -173,11 +230,32 @@ test("cloud onboard: public installer creates healthy sandbox with security chec const corporateCaProbe = await sandbox.execShell(SANDBOX_NAME, corporateCaMergeProbeScript(), { artifactName: "phase-2-corporate-ca-merge-probe", - env: env(), + env: testEnv(), timeoutMs: 60_000, }); expect(corporateCaProbe.exitCode, resultText(corporateCaProbe)).toBe(0); + progress.phase("collect scoped diagnostics from onboarded sandbox"); + const diagnosticsArchive = path.join(installCwd, "cloud-onboard-debug.tar.gz"); + const diagnostics = await host.command( + "bash", + [ + "-lc", + `nemoclaw debug --quick --sandbox ${shellQuote(SANDBOX_NAME)} --output ${shellQuote(diagnosticsArchive)}`, + ], + { + artifactName: "phase-3-scoped-diagnostics", + env: testEnv(), + timeoutMs: 60_000, + }, + ); + expect(diagnostics.exitCode, resultText(diagnostics)).toBe(0); + expect(fs.existsSync(diagnosticsArchive), "scoped diagnostics archive must exist").toBe(true); + expect( + fs.statSync(diagnosticsArchive).size, + "scoped diagnostics archive must be non-empty", + ).toBeGreaterThan(0); + progress.phase("run cloud inference and security checks"); const checkScripts = fs .readdirSync(CHECKS_DIR) @@ -186,9 +264,9 @@ test("cloud onboard: public installer creates healthy sandbox with security chec expect(checkScripts.length).toBeGreaterThan(0); for (const scriptName of checkScripts) { const result = await host.command("bash", [path.join(CHECKS_DIR, scriptName)], { - artifactName: `phase-3-check-${scriptName.replace(/\.sh$/, "")}`, + artifactName: `phase-4-check-${scriptName.replace(/\.sh$/, "")}`, cwd: REPO_ROOT, - env: env({ + env: testEnv({ ...hosted.env, CLOUD_EXPERIMENTAL_MODEL: hosted.model, COMPATIBLE_API_KEY: hosted.apiKey, @@ -203,6 +281,15 @@ test("cloud onboard: public installer creates healthy sandbox with security chec } progress.phase("remove cloud sandbox"); - await cleanup(host, sandbox, { label: "final-cleanup", verify: true }); - await artifacts.target.complete({ id: "cloud-onboard", status: "passed" }); + await cleanup(host, sandbox, { home: testHome, label: "final-cleanup", verify: true }); + await artifacts.target.complete({ + id: "cloud-onboard", + status: "passed", + credentialMigration: { + legacyFileRemoved: !fs.existsSync(legacyFile), + migratedProviderRegistered: providerNames.includes(hosted.providerName), + tamperedKeysExcluded: + !providerNames.includes("OPENSHELL_GATEWAY") && !providerNames.includes("NODE_OPTIONS"), + }, + }); }); diff --git a/test/e2e/live/credential-migration.test.ts b/test/e2e/live/credential-migration.test.ts deleted file mode 100644 index e0e173433d..0000000000 --- a/test/e2e/live/credential-migration.test.ts +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; -import { resultText } from "../fixtures/clients/command.ts"; -import type { HostCliClient } from "../fixtures/clients/host.ts"; -import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { testHomeEnvironment } from "../fixtures/environment-profiles.ts"; -import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; -import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; - -// Vitest test for the credential migration contract: a pre-gateway plaintext -// ~/.nemoclaw/credentials.json is staged only for allowlisted credential keys, -// a successful real onboard registers the migrated value with the OpenShell -// gateway, the plaintext file is removed after success, credentials list reads -// from the gateway, and secure unlink removes a planted symlink without touching -// its target. The repository secret is named NVIDIA_INFERENCE_API_KEY, but the -// hosted E2E service is the OpenAI-compatible inference-api.nvidia.com endpoint, -// so the migration contract stages that value as COMPATIBLE_API_KEY and expects -// the compatible-endpoint gateway provider. - -const DIST_CREDENTIAL_STORE = path.join(REPO_ROOT, "dist", "lib", "credentials", "store.js"); -const ONBOARD_TIMEOUT_MS = 30 * 60_000; -const INSTALL_TIMEOUT_MS = 10 * 60_000; -const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-cred-migration-${process.pid}`; -const CREDENTIAL_MIGRATION_MODEL = "openai/gpt-oss-120b"; -validateSandboxName(SANDBOX_NAME); - -type CommandResult = { stdout: string; stderr: string; exitCode: number | null }; - -function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return testHomeEnvironment(home, extra); -} - -async function bestEffortPreclean(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup probes are intentionally best-effort. Some paths fail before - // openshell is installed or before gateway/sandbox state exists. - } -} - -async function ensureOpenshellAvailable(host: HostCliClient, home: string): Promise { - const env = testEnv(home); - const current = await host.command( - "bash", - ["-lc", "command -v openshell && openshell --version"], - { - artifactName: "prereq-openshell-version", - env, - timeoutMs: 30_000, - }, - ); - if (current.exitCode === 0) return current; - - const install = await host.command( - "bash", - [path.join(REPO_ROOT, "scripts", "install-openshell.sh")], - { - artifactName: "prereq-install-openshell", - cwd: REPO_ROOT, - env, - timeoutMs: INSTALL_TIMEOUT_MS, - }, - ); - expect(install.exitCode, `install-openshell.sh failed\n${resultText(install)}`).toBe(0); - - const afterInstall = await host.command( - "bash", - ["-lc", "command -v openshell && openshell --version"], - { - artifactName: "prereq-openshell-version-after-install", - env: testEnv(home), - timeoutMs: 30_000, - }, - ); - expect( - afterInstall.exitCode, - `openshell missing after install\n${resultText(afterInstall)}`, - ).toBe(0); - return afterInstall; -} - -async function cleanupCredentialMigrationNemoClawSandbox( - host: HostCliClient, - home: string, -): Promise { - const env = testEnv(home); - const result = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy", - env, - redactionValues: [ - process.env.NVIDIA_INFERENCE_API_KEY ?? "", - process.env.COMPATIBLE_API_KEY ?? "", - ], - timeoutMs: 120_000, - }); - assertCleanupSucceededOrAbsent( - result, - /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( - resultText(result), - ), - `cleanup credential migration sandbox ${SANDBOX_NAME}`, - ); -} - -async function cleanupCredentialMigrationState(host: HostCliClient, home: string): Promise { - const env = testEnv(home); - await bestEffortPreclean(() => cleanupCredentialMigrationNemoClawSandbox(host, home)); - await bestEffortPreclean(() => - host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete", - env, - timeoutMs: 60_000, - }), - ); - await bestEffortPreclean(() => - host.command("openshell", ["forward", "stop", "18789"], { - artifactName: "cleanup-openshell-forward-stop", - env, - timeoutMs: 30_000, - }), - ); - await bestEffortPreclean(() => - host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-openshell-gateway-destroy", - env, - timeoutMs: 120_000, - }), - ); -} - -test("credential migration stages legacy file into gateway and removes plaintext safely", { - timeout: ONBOARD_TIMEOUT_MS + INSTALL_TIMEOUT_MS + 5 * 60_000, - meta: { - e2ePhases: [ - "validate legacy credential migration prerequisites", - "stage legacy plaintext credential", - "onboard from legacy credential file", - "verify gateway-backed credential registry", - "prove symlink-safe legacy file removal", - ], - }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - // Use the existing nightly secret as the legacy provider credential. The - // onboard child env below deliberately does not receive that credential, so - // the only source is ~/.nemoclaw/credentials.json — matching the retired - // shell lane's migration contract. - const hostedInference = requireHostedInferenceConfig(secrets, process.env, { - model: CREDENTIAL_MIGRATION_MODEL, - }); - const migratedCredentialValue = hostedInference.apiKey; - const { - [hostedInference.credentialEnv]: _omittedCredential, - ...hostedInferenceEnvWithoutCredential - } = hostedInference.env; - expect(fs.existsSync(CLI_ENTRYPOINT), "bin/nemoclaw.js missing").toBe(true); - expect( - fs.existsSync(DIST_CREDENTIAL_STORE), - "run `npm run build:cli` before this live test", - ).toBe(true); - - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for credential migration live E2E: ${resultText(docker)}`, - ); - } - skip("Docker is required for credential migration live E2E"); - } - - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cred-migration-")); - const nemoclawDir = path.join(home, ".nemoclaw"); - const legacyFile = path.join(nemoclawDir, "credentials.json"); - const cleanupEnv = testEnv(home); - cleanup.trackDisposable(`remove credential migration test home ${home}`, () => - fs.rmSync(home, { recursive: true, force: true }), - ); - cleanup.trackGateway(host, "nemoclaw", { - artifactName: "cleanup-openshell-gateway-destroy", - env: cleanupEnv, - timeoutMs: 120_000, - }); - cleanup.trackForward(host, 18789, { - artifactName: "cleanup-openshell-forward-stop", - env: cleanupEnv, - timeoutMs: 30_000, - }); - cleanup.trackDisposable(`delete credential migration OpenShell sandbox ${SANDBOX_NAME}`, () => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-sandbox-delete", - env: cleanupEnv, - timeoutMs: 60_000, - }), - ); - cleanup.trackDisposable(`destroy credential migration sandbox ${SANDBOX_NAME}`, () => - cleanupCredentialMigrationNemoClawSandbox(host, home), - ); - - await artifacts.target.declare({ - id: "credential-migration", - boundary: "real-onboard-openshell-gateway", - sandboxName: SANDBOX_NAME, - contracts: [ - "legacy credentials.json stages allowlisted provider keys into onboard env", - `successful onboard registers the migrated value with the ${hostedInference.providerName} OpenShell gateway provider`, - `${hostedInference.sourceSecretName} is migrated into the ${hostedInference.credentialEnv} provider credential`, - `onboard uses the ${hostedInference.provider} provider and ${hostedInference.endpointUrl} endpoint path`, - "successful onboard removes plaintext credentials.json", - "tampered non-credential keys do not become gateway providers", - "credentials list reads providers from the gateway, not disk", - "secure unlink removes a final-component symlink without touching its target", - ], - }); - - await ensureOpenshellAvailable(host, home); - await cleanupCredentialMigrationState(host, home); - - progress.phase("stage legacy plaintext credential"); - fs.rmSync(nemoclawDir, { recursive: true, force: true }); - fs.mkdirSync(nemoclawDir, { recursive: true, mode: 0o700 }); - fs.writeFileSync( - legacyFile, - JSON.stringify( - { - [hostedInference.credentialEnv]: migratedCredentialValue, - OPENSHELL_GATEWAY: "evil-gw-from-tampered-file", - NODE_OPTIONS: "--require=/tmp/evil.js", - }, - null, - 2, - ), - { mode: 0o600 }, - ); - - progress.phase("onboard from legacy credential file"); - const onboard = await host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { - artifactName: "onboard-from-legacy-credentials", - env: testEnv(home, { - ...hostedInferenceEnvWithoutCredential, - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_RECREATE_SANDBOX: "1", - }), - redactionValues: [migratedCredentialValue], - timeoutMs: ONBOARD_TIMEOUT_MS, - }); - const onboardText = resultText(onboard); - expect(onboard.exitCode, onboardText).toBe(0); - expect(onboardText).toContain( - "Staged 1 legacy credential(s) for migration to the OpenShell gateway.", - ); - expect(fs.existsSync(legacyFile), "legacy credentials.json must be removed after onboard").toBe( - false, - ); - - progress.phase("verify gateway-backed credential registry"); - const providers = await host.command( - "openshell", - ["-g", "nemoclaw", "provider", "list", "--names"], - { - artifactName: "gateway-provider-list", - env: testEnv(home), - timeoutMs: 60_000, - }, - ); - const providersText = resultText(providers); - expect(providers.exitCode, providersText).toBe(0); - const providerNames = providers.stdout - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => /^[a-zA-Z][a-zA-Z0-9_-]*$/.test(line)); - expect( - providerNames, - `expected migrated ${hostedInference.providerName} provider\n${providersText}`, - ).toContain(hostedInference.providerName); - expect(providerNames).not.toContain("OPENSHELL_GATEWAY"); - expect(providerNames).not.toContain("NODE_OPTIONS"); - - const credentialsList = await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { - artifactName: "nemoclaw-credentials-list", - env: testEnv(home), - redactionValues: [migratedCredentialValue], - timeoutMs: 60_000, - }); - const credentialsText = resultText(credentialsList); - expect(credentialsList.exitCode, credentialsText).toBe(0); - expect(credentialsText).toContain("Providers registered with the OpenShell gateway"); - expect( - fs.existsSync(legacyFile), - "credentials list must not recreate plaintext credentials.json", - ).toBe(false); - - progress.phase("prove symlink-safe legacy file removal"); - const victimFile = path.join(home, "victim.txt"); - const victimPayload = "important data the attacker should not touch"; - fs.writeFileSync(victimFile, victimPayload, { mode: 0o600 }); - fs.symlinkSync(victimFile, legacyFile); - - const unlink = await host.command( - "node", - [ - "-e", - `const { removeLegacyCredentialsFile } = require(${JSON.stringify(DIST_CREDENTIAL_STORE)}); removeLegacyCredentialsFile();`, - ], - { - artifactName: "remove-legacy-credentials-symlink", - env: testEnv(home), - timeoutMs: 30_000, - }, - ); - expect(unlink.exitCode, resultText(unlink)).toBe(0); - expect(fs.existsSync(legacyFile), "symlink at credentials path must be removed").toBe(false); - expect(fs.existsSync(victimFile), "symlink target must remain present").toBe(true); - expect(fs.readFileSync(victimFile, "utf-8")).toBe(victimPayload); - - await artifacts.target.complete({ - id: "credential-migration", - sandboxName: SANDBOX_NAME, - model: hostedInference.model || CREDENTIAL_MIGRATION_MODEL, - provider: hostedInference.providerName, - credentialEnv: hostedInference.credentialEnv, - providerNames, - assertions: { - onboardSucceeded: onboard.exitCode === 0, - migrationNoticeEmitted: onboardText.includes( - "Staged 1 legacy credential(s) for migration to the OpenShell gateway.", - ), - legacyFileRemovedAfterOnboard: !fs.existsSync(legacyFile), - migratedProviderRegistered: providerNames.includes(hostedInference.providerName), - tamperedKeysExcluded: - !providerNames.includes("OPENSHELL_GATEWAY") && !providerNames.includes("NODE_OPTIONS"), - credentialsListReadsGateway: credentialsText.includes( - "Providers registered with the OpenShell gateway", - ), - symlinkTargetUntouched: - fs.existsSync(victimFile) && fs.readFileSync(victimFile, "utf-8") === victimPayload, - }, - }); -}); diff --git a/test/e2e/live/credential-sanitization.test.ts b/test/e2e/live/credential-sanitization.test.ts deleted file mode 100644 index 6ddff2d26f..0000000000 --- a/test/e2e/live/credential-sanitization.test.ts +++ /dev/null @@ -1,409 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * - * Keep the replacement direct and small: local product-code assertions cover the - * sanitization/digest contract, then a real install/onboard verifies the - * sandbox filesystem does not expose credential artifacts or secret-shaped - * values. No new target framework, registry, or shared helper is introduced. - */ - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import YAML from "yaml"; -import { - isCredentialField, - isSensitiveFile, - sanitizeConfigFile, - stripCredentials, -} from "../../../src/lib/security/credential-filter.ts"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; -import { resultText } from "../fixtures/clients/command.ts"; -import type { HostCliClient } from "../fixtures/clients/host.ts"; -import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { testHomeEnvironment } from "../fixtures/environment-profiles.ts"; -import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; - -const BLUEPRINT_FILE = path.join(REPO_ROOT, "nemoclaw-blueprint", "blueprint.yaml"); -const SANDBOX_NAME = - process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-credential-sanitization-${process.pid}`; -const GATEWAY_PORT = "18080"; -const GATEWAY_NAME = `nemoclaw-${GATEWAY_PORT}`; -const INSTALL_TIMEOUT_MS = 45 * 60_000; -const SANDBOX_PROBE_TIMEOUT_MS = 120_000; -validateSandboxName(SANDBOX_NAME); - -const runCredentialSanitizationTest = test; - -type CommandText = { stdout: string; stderr: string }; - -type Blueprint = { - digest?: unknown; - components?: { sandbox?: { image?: unknown } }; -}; - -function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return testHomeEnvironment(home, { - NEMOCLAW_GATEWAY_PORT: GATEWAY_PORT, - OPENSHELL_GATEWAY: GATEWAY_NAME, - ...extra, - }); -} - -async function bestEffortPreclean(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup probes are intentionally best-effort: local failures can happen - // before install.sh has put OpenShell on this test HOME's PATH. - } -} - -async function cleanupCredentialSanitizationState( - host: HostCliClient, - home: string, -): Promise { - const env = testEnv(home); - await bestEffortPreclean(() => cleanupCredentialSanitizationNemoClawSandbox(host, home)); - await bestEffortPreclean(() => - host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete-credential-sanitization", - env, - timeoutMs: 60_000, - }), - ); - await bestEffortPreclean(() => - host.command("openshell", ["gateway", "destroy", "-g", GATEWAY_NAME], { - artifactName: "cleanup-openshell-gateway-destroy-credential-sanitization", - env, - timeoutMs: 120_000, - }), - ); -} - -async function cleanupCredentialSanitizationNemoClawSandbox( - host: HostCliClient, - home: string, -): Promise { - const result = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-credential-sanitization", - env: testEnv(home), - timeoutMs: 120_000, - }); - assertCleanupSucceededOrAbsent( - result, - /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( - resultText(result), - ), - `cleanup credential sanitization sandbox ${SANDBOX_NAME}`, - ); -} - -function removeSensitiveFiles(dir: string): void { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = path.join(dir, entry.name); - if (entry.isSymbolicLink()) continue; - if (entry.isDirectory()) { - removeSensitiveFiles(fullPath); - } else if (entry.isFile() && isSensitiveFile(entry.name)) { - fs.rmSync(fullPath, { force: true }); - } - } -} - -function assertLocalCredentialSanitizationContract(): void { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cred-sanitize-")); - try { - const bundleDir = path.join(tmp, "bundle", "openclaw"); - const authDir = path.join(bundleDir, "agents", "main", "agent"); - const workspaceDir = path.join(bundleDir, "workspace"); - fs.mkdirSync(authDir, { recursive: true }); - fs.mkdirSync(workspaceDir, { recursive: true }); - - const fakeNvidiaKey = "test-fake-nvidia-key-0000000000000000"; - const fakeGithubToken = "test-fake-github-token-1111111111111111"; - const fakeNpmToken = "test-fake-npm-token-2222222222222222"; - const fakeGatewayToken = "test-fake-gateway-token-333333333333"; - - fs.writeFileSync( - path.join(bundleDir, "openclaw.json"), - JSON.stringify( - { - agents: { - defaults: { - model: { primary: "nvidia/nemotron-3-super-120b-a12b" }, - workspace: workspaceDir, - }, - }, - gateway: { mode: "local", auth: { token: fakeGatewayToken } }, - nvidia: { apiKey: fakeNvidiaKey }, - mcpServers: { github: { env: { GITHUB_TOKEN: fakeGithubToken } } }, - }, - null, - 2, - ), - ); - fs.writeFileSync( - path.join(authDir, "auth-profiles.json"), - JSON.stringify({ - "github:pat": { token: fakeGithubToken }, - "npm:publish": { token: fakeNpmToken }, - }), - ); - fs.writeFileSync(path.join(workspaceDir, "project.md"), "# My Project\n"); - - sanitizeConfigFile(path.join(bundleDir, "openclaw.json")); - removeSensitiveFiles(bundleDir); - - const serialized = fs.readFileSync(path.join(bundleDir, "openclaw.json"), "utf8"); - expect(serialized).not.toContain(fakeNvidiaKey); - expect(serialized).not.toContain(fakeGithubToken); - expect(serialized).not.toContain(fakeGatewayToken); - expect(fs.existsSync(path.join(authDir, "auth-profiles.json"))).toBe(false); - - const sanitized = JSON.parse(serialized) as { - agents?: { defaults?: { model?: { primary?: string } } }; - nvidia?: { apiKey?: string }; - mcpServers?: { github?: { env?: { GITHUB_TOKEN?: string } } }; - gateway?: unknown; - }; - expect(sanitized.nvidia?.apiKey).toBe("[STRIPPED_BY_MIGRATION]"); - expect(sanitized.mcpServers?.github?.env?.GITHUB_TOKEN).toBe("[STRIPPED_BY_MIGRATION]"); - expect(sanitized.gateway).toBeUndefined(); - expect(sanitized.agents?.defaults?.model?.primary).toBe("nvidia/nemotron-3-super-120b-a12b"); - expect(fs.readFileSync(path.join(workspaceDir, "project.md"), "utf8")).toBe("# My Project\n"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } -} - -function assertCredentialFieldDetectionContract(): void { - const credentialFields = [ - "accessToken", - "refreshToken", - "privateKey", - "clientSecret", - "signingKey", - "bearerToken", - "sessionToken", - "authKey", - ]; - for (const field of credentialFields) { - expect(isCredentialField(field), `${field} should be credential-bearing`).toBe(true); - } - - const config = { - provider: Object.fromEntries( - credentialFields.map((field) => [field, `test-${field}-value`]), - ) as Record, - displayName: "should-be-preserved", - sortKey: "should-also-be-preserved", - modelName: "nvidia/nemotron-3-super-120b-a12b", - keyRef: { source: "env", id: "NVIDIA_INFERENCE_API_KEY" }, - description: "A secret garden (but not a real secret)", - tokenizer: "sentencepiece", - endpoint: "https://api.nvidia.com/v1", - sessionId: "abc-123", - accessLevel: "admin", - publicUrl: "https://example.com", - }; - - const sanitized = stripCredentials(config); - for (const value of Object.values(sanitized.provider)) { - expect(value).toBe("[STRIPPED_BY_MIGRATION]"); - } - expect(sanitized.displayName).toBe(config.displayName); - expect(sanitized.sortKey).toBe(config.sortKey); - expect(sanitized.modelName).toBe(config.modelName); - expect(sanitized.keyRef).toEqual(config.keyRef); - expect(sanitized.description).toBe(config.description); - expect(sanitized.tokenizer).toBe(config.tokenizer); - expect(sanitized.endpoint).toBe(config.endpoint); - expect(sanitized.sessionId).toBe(config.sessionId); - expect(sanitized.accessLevel).toBe(config.accessLevel); - expect(sanitized.publicUrl).toBe(config.publicUrl); -} - -function assertBlueprintDigestContract(): void { - const blueprint = YAML.parse(fs.readFileSync(BLUEPRINT_FILE, "utf8")) as Blueprint; - const topLevelDigest = typeof blueprint.digest === "string" ? blueprint.digest : ""; - expect(topLevelDigest).toMatch(/^sha256:[0-9a-f]{64}$/); - - const image = - typeof blueprint.components?.sandbox?.image === "string" - ? blueprint.components.sandbox.image - : ""; - const imageDigest = image.match(/@sha256:([0-9a-f]{64})$/)?.[1] ?? ""; - expect(`sha256:${imageDigest}`).toBe(topLevelDigest); -} - -async function assertSandboxCredentialBoundary( - sandbox: SandboxClient, - home: string, - apiKey: string, -): Promise { - const env = testEnv(home); - const authProbe = await sandbox.exec( - SANDBOX_NAME, - [ - "sh", - "-lc", - "find /sandbox -name auth-profiles.json -not -path '*/node_modules/*' -not -path '*/dist/*' -print 2>/dev/null | head -5", - ], - { - artifactName: "sandbox-auth-profiles-probe-credential-sanitization", - env, - timeoutMs: SANDBOX_PROBE_TIMEOUT_MS, - }, - ); - expect(authProbe.exitCode, resultText(authProbe)).toBe(0); - expect(authProbe.stdout.trim(), "auth-profiles.json must not be present in sandbox state").toBe( - "", - ); - - const secretProbe = await sandbox.exec( - SANDBOX_NAME, - [ - "sh", - "-lc", - "for dir in /sandbox/.openclaw /sandbox/.nemoclaw; do " + - '[ -d "$dir" ] || continue; ' + - "grep -rE 'nvapi-|ghp_|npm_' \"$dir\" 2>/dev/null " + - "| grep -v 'STRIPPED' " + - "| grep -v '/policies/' " + - "| grep -v '/plugin-runtime-deps/' " + - "| grep -Ev '/extensions/[^/]+/(dist|node_modules)/' " + - "| head -5 || true; " + - "done", - ], - { - artifactName: "sandbox-secret-pattern-probe-credential-sanitization", - env, - redactionValues: [apiKey], - timeoutMs: SANDBOX_PROBE_TIMEOUT_MS, - }, - ); - expect(secretProbe.exitCode, resultText(secretProbe)).toBe(0); - expect(secretProbe.stdout.trim(), "sandbox config must not contain secret-shaped tokens").toBe( - "", - ); -} - -runCredentialSanitizationTest( - "credential sanitization strips migration bundles and keeps sandbox secrets out of agent state", - { - timeout: INSTALL_TIMEOUT_MS + 10 * 60_000, - meta: { - e2ePhases: [ - "validate local credential sanitization contracts", - "check live credential boundary prerequisites", - "install credential-isolated OpenClaw sandbox", - "inspect sandbox for credential exposure", - ], - }, - }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - expect( - fs.existsSync(CLI_ENTRYPOINT), - "run `npm run build:cli` before live repo CLI targets", - ).toBe(true); - expect( - GATEWAY_PORT, - "the isolated test HOME must use a custom-port detached gateway instead of the host user service", - ).not.toBe("8080"); - const isolatedGatewayEnv = testEnv("/tmp/nemoclaw-credential-sanitization-contract"); - expect(isolatedGatewayEnv.NEMOCLAW_GATEWAY_PORT).toBe(GATEWAY_PORT); - expect(isolatedGatewayEnv.OPENSHELL_GATEWAY).toBe(GATEWAY_NAME); - - await artifacts.target.declare({ - id: "credential-sanitization", - boundary: "install-sh-onboard-and-sandbox-exec", - sandboxName: SANDBOX_NAME, - contracts: [ - "credential fields are stripped from migration bundle config while non-secret state survives", - "auth-profiles.json is removed from migration bundle state", - "pattern-based credential field detection strips token/key/secret/password suffixes without corrupting benign fields", - "shipped blueprint digest is non-empty and matches the pinned sandbox image digest", - "install.sh onboards a real Docker/OpenShell sandbox on the ubuntu-latest runner class", - "sandbox filesystem does not expose auth-profiles.json or secret-shaped token values in agent state", - ], - }); - - assertLocalCredentialSanitizationContract(); - assertCredentialFieldDetectionContract(); - assertBlueprintDigestContract(); - - progress.phase("check live credential boundary prerequisites"); - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-credential-sanitization", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for credential sanitization live E2E: ${resultText(docker)}`, - ); - } - skip("Docker is required for credential sanitization live E2E"); - } - - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cred-sanitization-home-")); - const cleanupEnv = testEnv(home); - cleanup.trackDisposable(`remove credential sanitization test home ${home}`, () => - fs.rmSync(home, { recursive: true, force: true }), - ); - cleanup.trackGateway(host, GATEWAY_NAME, { - artifactName: "cleanup-openshell-gateway-destroy-credential-sanitization", - env: cleanupEnv, - timeoutMs: 120_000, - }); - cleanup.trackDisposable( - `delete credential sanitization OpenShell sandbox ${SANDBOX_NAME}`, - () => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-sandbox-delete-credential-sanitization", - env: cleanupEnv, - timeoutMs: 60_000, - }), - ); - cleanup.trackDisposable(`destroy credential sanitization sandbox ${SANDBOX_NAME}`, () => - cleanupCredentialSanitizationNemoClawSandbox(host, home), - ); - - await cleanupCredentialSanitizationState(host, home); - - progress.phase("install credential-isolated OpenClaw sandbox"); - const install = await host.command( - "bash", - ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], - { - artifactName: "install-and-onboard-credential-sanitization", - cwd: REPO_ROOT, - env: testEnv(home, { - NVIDIA_INFERENCE_API_KEY: apiKey, - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_RECREATE_SANDBOX: "1", - }), - redactionValues: [apiKey], - timeoutMs: INSTALL_TIMEOUT_MS, - }, - ); - expect(install.exitCode, resultText(install)).toBe(0); - - progress.phase("inspect sandbox for credential exposure"); - const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { - artifactName: "nemoclaw-status-credential-sanitization", - env: testEnv(home), - timeoutMs: 60_000, - }); - expect(status.exitCode, resultText(status)).toBe(0); - - await assertSandboxCredentialBoundary(sandbox, home, apiKey); - }, -); diff --git a/test/e2e/live/diagnostics.test.ts b/test/e2e/live/diagnostics.test.ts deleted file mode 100644 index 70dc06ff3b..0000000000 --- a/test/e2e/live/diagnostics.test.ts +++ /dev/null @@ -1,453 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * Preserves the real boundaries: repo CLI/version, debug archive - * creation/extraction, credential redaction checks, install.sh/onboard, - * Docker/OpenShell sandbox registration, sandbox exec for openclaw.json, host - * status output, and gateway-backed credentials list/reset behavior. - */ - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/command.ts"; -import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { testHomeEnvironment } from "../fixtures/environment-profiles.ts"; -import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; -import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; - -const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-diag-${process.pid}`; -const DEBUG_QUICK_TIMEOUT_MS = 30_000; -const INSTALL_TIMEOUT_MS = 35 * 60_000; -const TEST_TIMEOUT_MS = 55 * 60_000; -validateSandboxName(SANDBOX_NAME); - -type RawCommandResult = { - status: number | null; - signal: NodeJS.Signals | null; - stdout: string; - stderr: string; - error?: Error; -}; - -function rawResultText(result: Pick): string { - return [result.stdout, result.stderr].filter(Boolean).join("\n"); -} - -function redactForAssertion(text: string, apiKey: string): string { - return text - .split(apiKey) - .join("[REDACTED]") - .replace(/nvapi-[A-Za-z0-9_-]{10,}/g, ""); -} - -function runRawNodeCliForLeakAssertion(args: string[], env: NodeJS.ProcessEnv): RawCommandResult { - const result = spawnSync("node", [CLI_ENTRYPOINT, ...args], { - cwd: REPO_ROOT, - encoding: "utf8", - env, - killSignal: "SIGKILL", - timeout: 60_000, - }); - return { - status: result.status, - signal: result.signal, - stdout: result.stdout ?? "", - stderr: result.stderr ?? "", - error: result.error, - }; -} - -function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return testHomeEnvironment(home, { - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT: "1", - ...extra, - }); -} - -async function preCleanBestEffort(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup probes are intentionally best-effort so they do not mask the - // primary diagnostics assertion. - } -} - -function assertNoSecretInExtractedArchive(extractDir: string, apiKey: string): void { - const files: string[] = []; - const visit = (dir: string): void => { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) visit(fullPath); - else if (entry.isFile()) files.push(fullPath); - } - }; - visit(extractDir); - - const leakedFiles: string[] = []; - const patternLeaks: string[] = []; - const apiKeyBytes = Buffer.from(apiKey, "utf8"); - for (const file of files) { - const content = fs.readFileSync(file); - const text = content.toString("utf8"); - if (content.includes(apiKeyBytes)) leakedFiles.push(path.relative(extractDir, file)); - if (/nvapi-[A-Za-z0-9_-]{10,}/.test(text)) patternLeaks.push(path.relative(extractDir, file)); - } - - expect(leakedFiles, "debug archive must not contain the exact NVIDIA_INFERENCE_API_KEY").toEqual( - [], - ); - expect(patternLeaks, "debug archive must not contain nvapi-shaped credentials").toEqual([]); -} - -test("diagnostics CLI creates sanitized archives and validates sandbox/credential diagnostics", { - timeout: TEST_TIMEOUT_MS, - meta: { - e2ePhases: [ - "validate diagnostics runtime prerequisites", - "exercise quick debug archive", - "install diagnostics sandbox", - "inspect sanitized full and scoped archives", - "validate sandbox status and config", - "audit and reset gateway credentials", - ], - }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - expect( - fs.existsSync(CLI_ENTRYPOINT), - "run `npm run build:cli` before live repo CLI targets", - ).toBe(true); - - const hosted = requireHostedInferenceConfig(secrets); - const apiKey = hosted.apiKey; - await artifacts.target.declare({ - id: "diagnostics", - boundary: "debug-archive-install-sh-docker-openshell-sandbox-exec-credentials", - sandboxName: SANDBOX_NAME, - contracts: [ - "nemoclaw --version exits zero and prints semver", - "nemoclaw debug --quick creates a non-empty archive within the quick timeout", - "nemoclaw debug --output creates an extractable archive without NVIDIA credential values", - "debug --sandbox accepts a registered sandbox and rejects an unknown sandbox without a partial archive", - "sandbox openclaw.json is readable through real OpenShell sandbox exec and host status includes model data", - "credentials list hides secret values and credentials reset removes an explicitly detached provider credential from the gateway", - ], - }); - - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-diagnostics", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for diagnostics live E2E: ${resultText(docker)}`); - } - skip("Docker is required for diagnostics live E2E"); - } - - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-diagnostics-home-")); - const cleanupEnv = testEnv(home); - cleanup.trackDisposable("remove diagnostics home", () => { - fs.rmSync(home, { recursive: true, force: true }); - }); - cleanup.trackGateway(host, "nemoclaw", { - artifactName: "cleanup-openshell-gateway-destroy-diagnostics", - env: cleanupEnv, - redactionValues: [apiKey], - timeoutMs: 120_000, - }); - cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-sandbox-delete-diagnostics", - env: cleanupEnv, - redactionValues: [apiKey], - timeoutMs: 60_000, - }), - ); - cleanup.trackSandbox(host, SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-diagnostics", - env: cleanupEnv, - redactionValues: [apiKey], - timeoutMs: 120_000, - }); - - const env = testEnv(home, hosted.env); - await preCleanBestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "pre-cleanup-nemoclaw-destroy-diagnostics", - env, - redactionValues: [apiKey], - timeoutMs: 120_000, - }), - ); - await preCleanBestEffort(() => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "pre-cleanup-openshell-sandbox-delete-diagnostics", - env, - timeoutMs: 60_000, - }), - ); - - progress.phase("exercise quick debug archive"); - const version = await host.command("node", [CLI_ENTRYPOINT, "--version"], { - artifactName: "diagnostics-nemoclaw-version", - env: testEnv(home), - timeoutMs: 30_000, - }); - expect(version.exitCode, resultText(version)).toBe(0); - expect(resultText(version)).toMatch(/\d+\.\d+\.\d+/); - - const quickDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-debug-quick-")); - const quickArchive = path.join(quickDir, "quick-debug.tar.gz"); - const quickStartedAt = Date.now(); - const quick = await host.command( - "node", - [CLI_ENTRYPOINT, "debug", "--quick", "--output", quickArchive], - { - artifactName: "diagnostics-debug-quick", - env: testEnv(home, { NEMOCLAW_SANDBOX_NAME: "" }), - timeoutMs: DEBUG_QUICK_TIMEOUT_MS, - }, - ); - const quickElapsedMs = Date.now() - quickStartedAt; - expect(quick.exitCode, resultText(quick)).toBe(0); - expect(fs.existsSync(quickArchive), "debug --quick must create an archive").toBe(true); - expect(fs.statSync(quickArchive).size, "debug --quick archive must be non-empty").toBeGreaterThan( - 0, - ); - expect( - quickElapsedMs, - "debug --quick must complete within the legacy 30s process timeout plus harness scheduling grace", - ).toBeLessThanOrEqual(DEBUG_QUICK_TIMEOUT_MS + 5_000); - - progress.phase("install diagnostics sandbox"); - const install = await host.command( - "bash", - ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], - { - artifactName: "install-and-onboard-diagnostics", - cwd: REPO_ROOT, - env, - redactionValues: [apiKey], - timeoutMs: INSTALL_TIMEOUT_MS, - }, - ); - expect(install.exitCode, resultText(install)).toBe(0); - - progress.phase("inspect sanitized full and scoped archives"); - const fullDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-debug-full-")); - const fullArchive = path.join(fullDir, "debug-full.tar.gz"); - const extractDir = path.join(fullDir, "extracted"); - fs.mkdirSync(extractDir, { recursive: true }); - const fullDebug = await host.command("node", [CLI_ENTRYPOINT, "debug", "--output", fullArchive], { - artifactName: "diagnostics-debug-full", - env, - redactionValues: [apiKey], - timeoutMs: 180_000, - }); - expect(fullDebug.exitCode, resultText(fullDebug)).toBe(0); - expect(fs.existsSync(fullArchive), "debug --output must create an archive").toBe(true); - expect(fs.statSync(fullArchive).size, "debug --output archive must be non-empty").toBeGreaterThan( - 0, - ); - const extract = await host.command("tar", ["xzf", fullArchive, "-C", extractDir], { - artifactName: "diagnostics-debug-full-extract", - env: testEnv(home), - timeoutMs: 60_000, - }); - expect(extract.exitCode, resultText(extract)).toBe(0); - assertNoSecretInExtractedArchive(extractDir, apiKey); - - const knownArchive = path.join(fullDir, "known-sandbox.tar.gz"); - const knownSandboxDebug = await host.command( - "node", - [CLI_ENTRYPOINT, "debug", "--quick", "--sandbox", SANDBOX_NAME, "--output", knownArchive], - { - artifactName: "diagnostics-debug-known-sandbox", - env, - redactionValues: [apiKey], - timeoutMs: DEBUG_QUICK_TIMEOUT_MS, - }, - ); - expect(knownSandboxDebug.exitCode, resultText(knownSandboxDebug)).toBe(0); - expect(fs.existsSync(knownArchive), "registered --sandbox must create an archive").toBe(true); - expect( - fs.statSync(knownArchive).size, - "registered --sandbox archive must be non-empty", - ).toBeGreaterThan(0); - - const missingName = `nemoclaw-e2e-missing-${process.pid}-${Date.now()}`; - const missingArchive = path.join(fullDir, "unknown-sandbox.tar.gz"); - const unknownSandboxDebug = await host.command( - "node", - [CLI_ENTRYPOINT, "debug", "--quick", "--sandbox", missingName, "--output", missingArchive], - { - artifactName: "diagnostics-debug-unknown-sandbox", - env, - redactionValues: [apiKey], - timeoutMs: DEBUG_QUICK_TIMEOUT_MS, - }, - ); - const unknownText = resultText(unknownSandboxDebug); - expect(unknownSandboxDebug.exitCode, unknownText).not.toBe(0); - expect(unknownText).toContain(missingName); - expect(unknownText).toMatch(/not registered/i); - expect(fs.existsSync(missingArchive), "unknown --sandbox must not leave a partial archive").toBe( - false, - ); - - progress.phase("validate sandbox status and config"); - const config = await sandbox.exec( - SANDBOX_NAME, - ["sh", "-lc", "cat /sandbox/.openclaw/openclaw.json"], - { - artifactName: "diagnostics-sandbox-openclaw-config", - env, - redactionValues: [apiKey], - timeoutMs: 60_000, - }, - ); - expect(config.exitCode, resultText(config)).toBe(0); - expect(config.stdout.trim(), "openclaw.json must be readable inside sandbox").not.toBe(""); - - const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { - artifactName: "diagnostics-nemoclaw-status", - env, - redactionValues: [apiKey], - timeoutMs: 60_000, - }); - expect(status.exitCode, resultText(status)).toBe(0); - expect(resultText(status)).toMatch(/Model/i); - - progress.phase("audit and reset gateway credentials"); - const rawCredentialsList = runRawNodeCliForLeakAssertion(["credentials", "list"], env); - const credentialsOutput = rawResultText(rawCredentialsList); - const credentialsStdout = rawCredentialsList.stdout; - expect(rawCredentialsList.status, redactForAssertion(credentialsOutput, apiKey)).toBe(0); - expect( - credentialsOutput.includes(apiKey), - "credentials list must not expose the exact NVIDIA_INFERENCE_API_KEY", - ).toBe(false); - expect( - /nvapi-[A-Za-z0-9_-]{10,}/.test(credentialsOutput), - "credentials list must not expose nvapi-shaped values", - ).toBe(false); - expect( - credentialsStdout.includes(hosted.providerName) || - /No provider credentials registered/i.test(credentialsStdout), - ).toBe(true); - - await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { - artifactName: "diagnostics-credentials-list", - env, - redactionValues: [apiKey], - timeoutMs: 60_000, - }); - - let credentialsResetExercised = false; - let postResetCredentialsListRedacted = false; - let providerCredentialAbsentBeforeReset = false; - let providerDetachedBeforeReset = false; - if (credentialsStdout.includes(hosted.providerName)) { - credentialsResetExercised = true; - const detach = await sandbox.openshell( - ["sandbox", "provider", "detach", SANDBOX_NAME, hosted.providerName], - { - artifactName: "diagnostics-inference-provider-detach-before-credentials-reset", - env, - redactionValues: [apiKey], - timeoutMs: 60_000, - }, - ); - expect(detach.exitCode, resultText(detach)).toBe(0); - - const providersAfterDetach = await sandbox.openshell( - ["sandbox", "provider", "list", SANDBOX_NAME], - { - artifactName: "diagnostics-inference-providers-after-detach", - env, - redactionValues: [apiKey], - timeoutMs: 60_000, - }, - ); - expect(providersAfterDetach.exitCode, resultText(providersAfterDetach)).toBe(0); - expect(providersAfterDetach.stdout, resultText(providersAfterDetach)).not.toContain( - hosted.providerName, - ); - providerDetachedBeforeReset = true; - - const reset = await host.command( - "node", - [CLI_ENTRYPOINT, "credentials", "reset", hosted.providerName, "--yes"], - { - artifactName: "diagnostics-credentials-reset", - env, - redactionValues: [apiKey], - timeoutMs: 60_000, - }, - ); - expect(reset.exitCode, resultText(reset)).toBe(0); - expect(reset.stdout, resultText(reset)).toContain(`Removed provider '${hosted.providerName}'`); - - const rawPostResetList = runRawNodeCliForLeakAssertion(["credentials", "list"], env); - const postResetOutput = rawResultText(rawPostResetList); - expect(rawPostResetList.status, redactForAssertion(postResetOutput, apiKey)).toBe(0); - expect( - rawPostResetList.stdout.includes(hosted.providerName), - redactForAssertion(postResetOutput, apiKey), - ).toBe(false); - expect( - postResetOutput.includes(apiKey), - "post-reset credentials list must not expose the exact NVIDIA_INFERENCE_API_KEY", - ).toBe(false); - expect( - /nvapi-[A-Za-z0-9_-]{10,}/.test(postResetOutput), - "post-reset credentials list must not expose nvapi-shaped values", - ).toBe(false); - postResetCredentialsListRedacted = !postResetOutput.includes(apiKey); - - await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { - artifactName: "diagnostics-credentials-list-after-reset", - env, - redactionValues: [apiKey], - timeoutMs: 60_000, - }); - } else { - providerCredentialAbsentBeforeReset = true; - await artifacts.writeJson("credentials-reset.skip.json", { - provider: hosted.providerName, - reason: `credentials list reported no ${hosted.providerName} provider credential after install/onboard`, - acceptedNoProviderStore: /No provider credentials registered/i.test(credentialsStdout), - }); - } - - await artifacts.target.complete({ - id: "diagnostics", - sandboxName: SANDBOX_NAME, - model: hosted.model, - assertions: { - versionPrintedSemver: /\d+\.\d+\.\d+/.test(resultText(version)), - quickDebugArchiveCreated: fs.existsSync(quickArchive) && fs.statSync(quickArchive).size > 0, - fullDebugArchiveCreated: fs.existsSync(fullArchive) && fs.statSync(fullArchive).size > 0, - fullDebugArchiveSanitized: true, - registeredSandboxDebugAccepted: knownSandboxDebug.exitCode === 0, - unknownSandboxDebugRejected: unknownSandboxDebug.exitCode !== 0, - sandboxConfigReadable: config.exitCode === 0 && config.stdout.trim().length > 0, - statusShowsModel: /Model/i.test(resultText(status)), - credentialsListRedacted: !credentialsOutput.includes(apiKey), - credentialsResetExercised, - providerCredentialAbsentBeforeReset, - providerDetachedBeforeReset, - postResetCredentialsListRedacted, - }, - }); -}); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 672f06c592..d0162ee1f2 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -213,20 +213,6 @@ "test/e2e/support/e2e-clients.test.ts" ] }, - { - "live": "test/e2e/live/credential-migration.test.ts", - "fast": [ - "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" - ] - }, - { - "live": "test/e2e/live/credential-sanitization.test.ts", - "fast": [ - "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" - ] - }, { "live": "test/e2e/live/cron-preflight-inference-local.test.ts", "fast": [ @@ -241,13 +227,6 @@ "test/e2e/support/e2e-clients.test.ts" ] }, - { - "live": "test/e2e/live/diagnostics.test.ts", - "fast": [ - "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" - ] - }, { "live": "test/e2e/live/double-onboard.test.ts", "fast": [ diff --git a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts index 035b907e6e..a364403179 100644 --- a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts +++ b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts @@ -175,11 +175,11 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => { it("rejects alias, ordering, and no-image exemption drift", () => { const errors = validateMutation((workflow) => { const canonicalAuth = namedStep(workflow.jobs.live, AUTH_STEP_NAME)!; - const diagnosticsSteps = workflow.jobs.diagnostics.steps!; - const diagnosticsAuthIndex = diagnosticsSteps.indexOf( - namedStep(workflow.jobs.diagnostics, AUTH_STEP_NAME)!, + const cloudInferenceSteps = workflow.jobs["cloud-inference"].steps!; + const cloudInferenceAuthIndex = cloudInferenceSteps.indexOf( + namedStep(workflow.jobs["cloud-inference"], AUTH_STEP_NAME)!, ); - diagnosticsSteps[diagnosticsAuthIndex] = { + cloudInferenceSteps[cloudInferenceAuthIndex] = { ...canonicalAuth, env: { ...canonicalAuth.env }, }; @@ -196,7 +196,7 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => { expect(errors).toEqual( expect.arrayContaining([ - "diagnostics Docker Hub auth must reuse the canonical workflow alias", + "cloud-inference Docker Hub auth must reuse the canonical workflow alias", "messaging-compatible-endpoint Docker Hub auth must run immediately after checkout", "shared-e2e no-image job must not receive Docker Hub authentication", ]), diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index fcb3f52a72..3592df97c5 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -240,7 +240,7 @@ describe("E2E operations workflow boundary", () => { CORRELATION_ID: "123e4567-e89b-42d3-a456-426614174000", GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "unused", - JOBS: "credential-sanitization", + JOBS: "cloud-inference", PLAN_HASH: "c".repeat(64), PR_NUMBER: "42", RUN_ATTEMPT: "1", @@ -273,7 +273,7 @@ describe("E2E operations workflow boundary", () => { conclusion: null, details_url: "https://github.com/NVIDIA/NemoClaw/runs/17", output: { - summary: `Risk plan ${planHash} selected jobs: credential-sanitization; targets: none. Child run: https://github.com/NVIDIA/NemoClaw/actions/runs/23.`, + summary: `Risk plan ${planHash} selected jobs: cloud-inference; targets: none. Child run: https://github.com/NVIDIA/NemoClaw/actions/runs/23.`, }, }); const result = spawnSync( @@ -299,7 +299,7 @@ describe("E2E operations workflow boundary", () => { FAKE_CHECK: check, GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "unused", - JOBS: "credential-sanitization", + JOBS: "cloud-inference", PLAN_HASH: planHash, PR_NUMBER: "42", RUN_ATTEMPT: "1", diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 153ea264c9..52448406ac 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -1062,7 +1062,6 @@ jobs: "step 'Run double-onboard live Vitest test' run script must not interpolate dispatch inputs directly", "workflow missing hermes-e2e job", "workflow missing skill-agent job", - "workflow missing diagnostics job", "workflow missing model-router-provider-routed-inference job", "workflow missing snapshot-commands job", "report-to-pr job must wait for live", @@ -1401,69 +1400,6 @@ jobs: } }); - // source-shape-contract: security -- Mutates the shipped diagnostics job to reject secret and Docker auth leakage - it("rejects diagnostics workflow-boundary drift for secret and Docker auth handling", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); - const workflowPath = path.join(tmp, "workflow.yaml"); - const workflow = readWorkflow() as { - jobs: Record< - string, - { env?: Record; steps: Array> } - >; - }; - const job = workflow.jobs["diagnostics"]; - expect(job).toBeDefined(); - expect(job.steps).toEqual(expect.any(Array)); - job.env = { - ...job.env, - DOCKER_CONFIG: "${{ github.workspace }}/.docker-config-diagnostics", - NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - GITHUB_TOKEN: "${{ github.token }}", - }; - const prepareIndex = job.steps.findIndex((step) => step.name === "Prepare E2E workspace"); - expect(prepareIndex).toBeGreaterThan(0); - job.steps.splice(prepareIndex, 0, { - name: "Authenticate to Docker Hub", - env: { - DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", - DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", - }, - run: 'docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin', - }); - const runStep = job.steps.find((step) => step.name === "Run diagnostics live test"); - expect(runStep).toBeDefined(); - runStep!.run = `${runStep!.run}\necho "\${{ inputs.jobs }}"`; - const uploadStep = job.steps.find((step) => step.name === "Upload diagnostics artifacts"); - expect(uploadStep).toBeDefined(); - uploadStep!.with = { - ...((uploadStep!.with as Record) ?? {}), - "include-hidden-files": true, - "retention-days": 1, - }; - fs.writeFileSync(workflowPath, YAML.stringify(workflow)); - - try { - const errors = validateE2eWorkflowBoundary(workflowPath); - expect(errors).toEqual( - expect.arrayContaining([ - "diagnostics job must not expose Docker auth to branch-controlled steps", - "diagnostics job env must not include DOCKER_CONFIG", - "diagnostics job env must not include NVIDIA_INFERENCE_API_KEY", - "diagnostics job env must not include GITHUB_TOKEN", - "diagnostics image-consuming job must have exactly one Docker Hub auth step", - "diagnostics step 'Authenticate to Docker Hub' env must not include DOCKERHUB_USERNAME", - "diagnostics step 'Authenticate to Docker Hub' env must not include DOCKERHUB_TOKEN", - "diagnostics step 'Authenticate to Docker Hub' must not authenticate or interpolate Docker Hub secrets", - "step 'Run diagnostics live test' run script must not interpolate dispatch inputs directly", - "diagnostics upload-e2e-artifacts invocation must not override its contract", - "diagnostics upload-e2e-artifacts must use the action defaults", - ]), - ); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - it("rejects matrix generation that bypasses the planner CI-output mode", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/test/e2e/support/live-vitest-invocation.test.ts b/test/e2e/support/live-vitest-invocation.test.ts index eae744753d..69b40fe2a2 100644 --- a/test/e2e/support/live-vitest-invocation.test.ts +++ b/test/e2e/support/live-vitest-invocation.test.ts @@ -144,7 +144,7 @@ describe("resolveLiveSelector (#6901)", () => { it("does not infer selectors for unrelated live tests", () => { expect( - resolveLiveSelector("test/e2e/live/diagnostics.test.ts", undefined, { + resolveLiveSelector("test/e2e/live/cloud-inference.test.ts", undefined, { NEMOCLAW_MCP_BRIDGE_AGENT: "hermes", }), ).toBeUndefined(); @@ -175,14 +175,14 @@ describe("buildLiveVitestArgs (#6961)", () => { it("omits the selector arguments for a single-file target", () => { expect( buildLiveVitestArgs({ - testPath: "test/e2e/live/diagnostics.test.ts", + testPath: "test/e2e/live/cloud-inference.test.ts", }), ).toEqual([ "vitest", "run", "--project", "e2e-live", - "test/e2e/live/diagnostics.test.ts", + "test/e2e/live/cloud-inference.test.ts", "--silent=false", "--reporter=default", `--reporter=${RISK_SIGNAL_REPORTER}`, @@ -207,7 +207,7 @@ describe("buildLiveVitestArgs (#6961)", () => { }); describe("runLiveVitestCommand (#6961)", () => { - const validArgs = ["run", "--test-path", "test/e2e/live/diagnostics.test.ts"]; + const validArgs = ["run", "--test-path", "test/e2e/live/cloud-inference.test.ts"]; it.each([ ["child status", { status: 7, signal: null }, 7], @@ -228,7 +228,7 @@ describe("runLiveVitestCommand (#6961)", () => { "run", "--project", "e2e-live", - "test/e2e/live/diagnostics.test.ts", + "test/e2e/live/cloud-inference.test.ts", "--silent=false", "--reporter=default", `--reporter=${RISK_SIGNAL_REPORTER}`, @@ -267,7 +267,7 @@ describe("runLiveVitestCommand (#6961)", () => { it.each([ [ "unknown option", - ["run", "--test-path", "test/e2e/live/diagnostics.test.ts", "--selctor", "^x$"], + ["run", "--test-path", "test/e2e/live/cloud-inference.test.ts", "--selctor", "^x$"], ], ["bare selector", [...validArgs, "--selector"]], ])("rejects an %s before spawning Vitest", (_label, args) => { diff --git a/test/e2e/support/retired-selector-compatibility.test.ts b/test/e2e/support/retired-selector-compatibility.test.ts index ff2811941b..1b51c56a9e 100644 --- a/test/e2e/support/retired-selector-compatibility.test.ts +++ b/test/e2e/support/retired-selector-compatibility.test.ts @@ -17,6 +17,9 @@ const EXPECTED_SHA = "a".repeat(40); const PLAN_HASH = "b".repeat(64); const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; const REPLACEMENT_FILES = [ + "src/lib/security/credential-filter-secret-patterns.test.ts", + "test/credential-migration-reconciliation.test.ts", + "test/package-contract/cli/debug-cli-command.test.ts", "test/package-contract/cli/public-cli-contracts.test.ts", "test/gateway-drift-preflight.test.ts", "test/gateway-health-honest.test.ts", @@ -56,11 +59,11 @@ describe("retired E2E selector compatibility", () => { it("selects only IDs absent from an SHA-bound candidate inventory (#7616)", () => { expect( selectedRetiredControllerJobs({ - allowedJobs: ["cloud-onboard", "docs-validation"], + allowedJobs: ["cloud-onboard", "credential-sanitization", "docs-validation"], expectedSha: EXPECTED_SHA, - jobs: "cloud-onboard,docs-validation,gateway-health-honest", + jobs: "cloud-onboard,credential-sanitization,diagnostics,docs-validation,gateway-health-honest", }), - ).toEqual(["gateway-health-honest"]); + ).toEqual(["diagnostics", "gateway-health-honest"]); expect( selectedRetiredControllerJobs({ allowedJobs: ["cloud-onboard"], @@ -89,9 +92,10 @@ describe("retired E2E selector compatibility", () => { expect(selected).toEqual([...RETIRED_CONTROLLER_SELECTOR_IDS].sort()); expect(commands).toEqual([ - "npx vitest run --project integration test/credentials.test.ts test/gateway-drift-preflight.test.ts test/gateway-health-honest.test.ts", + "npx vitest run --project cli src/lib/security/credential-filter-secret-patterns.test.ts", + "npx vitest run --project integration test/credential-migration-reconciliation.test.ts test/credentials.test.ts test/gateway-drift-preflight.test.ts test/gateway-health-honest.test.ts", "npx vitest run --project installer-integration test/install-openshell-version-pin.test.ts", - "npx vitest run --project package-contract test/package-contract/cli/public-cli-contracts.test.ts test/package-contract/onboard/invalid-nvidia-key.test.ts", + "npx vitest run --project package-contract test/package-contract/cli/debug-cli-command.test.ts test/package-contract/cli/public-cli-contracts.test.ts test/package-contract/onboard/invalid-nvidia-key.test.ts", ]); expect(fs.readFileSync(target.output, "utf8")).toBe("selected=true\n"); for (const id of selected) { diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index b81f1a971f..1320517983 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -174,8 +174,8 @@ describe("upload-e2e-artifacts workflow boundary", () => { it("rejects default, explicit-exception, caller-key, and caller-if drift", () => { const workflow = mutableWorkflow(); - const defaultJob = workflow.jobs["credential-migration"]; - uploadStep(defaultJob).with = { name: "e2e-credential-migration" }; + const defaultJob = workflow.jobs["sessions-agents-cli"]; + uploadStep(defaultJob).with = { name: "e2e-sessions-agents-cli" }; defaultJob.env!.E2E_TARGET_ID = "not a selector id"; uploadStep(workflow.jobs["hermes-slack"]).with!.path = "e2e-artifacts/live/hermes-slack/"; @@ -195,9 +195,9 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "credential-migration upload-e2e-artifacts invocation must not override its contract", - "credential-migration upload-e2e-artifacts must use the action defaults", - "credential-migration default upload caller must declare a valid E2E_TARGET_ID", + "sessions-agents-cli upload-e2e-artifacts invocation must not override its contract", + "sessions-agents-cli upload-e2e-artifacts must use the action defaults", + "sessions-agents-cli default upload caller must declare a valid E2E_TARGET_ID", "hermes-slack upload-e2e-artifacts must preserve its explicit name/path contract", "network-policy upload-e2e-artifacts must preserve its explicit name/path contract", "common-egress-agent upload-e2e-artifacts must preserve its explicit name/path contract", @@ -228,14 +228,14 @@ describe("upload-e2e-artifacts workflow boundary", () => { it("derives execution jobs even when a marker and its upload disappear together", () => { const workflow = mutableWorkflow(); - const removedJob = workflow.jobs["credential-sanitization"]; + const removedJob = workflow.jobs["sessions-agents-cli"]; delete removedJob.env!.E2E_JOB; removedJob.steps = removedJob.steps!.filter( (step) => step.uses !== UPLOAD_E2E_ARTIFACTS_ACTION, ); expect(validateUploadE2eArtifactsInvocations(workflow)).toContain( - "credential-sanitization must use upload-e2e-artifacts exactly once", + "sessions-agents-cli must use upload-e2e-artifacts exactly once", ); }); }); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index e38f6398f1..5c83dfb8ac 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -146,7 +146,7 @@ describe("E2E workflow plan", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cli-")); const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); - const activeJobs = "cloud-onboard,credential-sanitization,security-posture"; + const activeJobs = "cloud-onboard,security-posture"; const plan = buildE2eWorkflowPlan({ jobs: activeJobs }); try { const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { diff --git a/test/onboard-inference-reconciliation.test.ts b/test/onboard-inference-reconciliation.test.ts index cbffc310ce..a44375fbd1 100644 --- a/test/onboard-inference-reconciliation.test.ts +++ b/test/onboard-inference-reconciliation.test.ts @@ -934,51 +934,4 @@ console.log(JSON.stringify({ ); }); }); - it("migrates a legacy credentials.json into env so setupInference can register the provider", async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-resume-cred-")); - const legacyDir = path.join(tmpDir, ".nemoclaw"); - const legacyFile = path.join(legacyDir, "credentials.json"); - fs.mkdirSync(legacyDir, { recursive: true, mode: 0o700 }); - fs.writeFileSync( - legacyFile, - JSON.stringify({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-STORED-KEY" }), - { mode: 0o600 }, - ); - const credentialEnv = - require("../src/lib/onboard/credential-env") as typeof import("../src/lib/onboard/credential-env.js"); - try { - await withProcessEnv({ HOME: tmpDir, OPENAI_API_KEY: undefined }, async () => { - const harness = createDirectSetupInferenceHarness({ - runOpenshell: (args) => - args.slice(0, 2).join(" ") === "provider get" - ? { status: 0, stdout: "", stderr: "" } - : undefined, - overrides: { hydrateCredentialEnv: credentialEnv.hydrateCredentialEnv }, - }); - - await harness.setupInference( - "test-box", - "gpt-5.4", - "openai-api", - "https://api.openai.com/v1", - "OPENAI_API_KEY", - ); - - assert.equal(process.env.OPENAI_API_KEY, "sk-TEST-NOT-A-REAL-STORED-KEY"); - assert.equal( - fs.existsSync(legacyFile), - true, - "legacy credentials.json must survive the staging-only hydrate path", - ); - const providerUpdate = harness.commands.find((entry) => - entry.command.includes("provider update -g nemoclaw openai-api"), - ); - assert.ok(providerUpdate, "expected provider update command"); - assert.equal(providerUpdate.env?.OPENAI_API_KEY, "sk-TEST-NOT-A-REAL-STORED-KEY"); - assert.doesNotMatch(providerUpdate.command, /sk-TEST-NOT-A-REAL-STORED-KEY/); - }); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); }); diff --git a/test/package-contract/cli/debug-cli-command.test.ts b/test/package-contract/cli/debug-cli-command.test.ts new file mode 100644 index 0000000000..2abcaa85e0 --- /dev/null +++ b/test/package-contract/cli/debug-cli-command.test.ts @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.join(import.meta.dirname, "../../.."); +const CLI_PATH = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +const NVIDIA_TEST_SECRET = "nvapi-TEST-NOT-A-REAL-DIAGNOSTIC-KEY"; +const GITHUB_TEST_SECRET = "ghp_TEST_NOT_A_REAL_DIAGNOSTIC_TOKEN"; + +type CliResult = { + status: number | null; + output: string; +}; + +function runCli(args: string[], env: NodeJS.ProcessEnv): CliResult { + const result = spawnSync(process.execPath, [CLI_PATH, ...args], { + cwd: REPO_ROOT, + encoding: "utf8", + env, + killSignal: "SIGKILL", + timeout: 30_000, + }); + expect(result.error).toBeUndefined(); + return { + status: result.status, + output: `${result.stdout ?? ""}${result.stderr ?? ""}`, + }; +} + +function writeExecutable(filePath: string, lines: string[]): void { + fs.writeFileSync(filePath, `${lines.join("\n")}\n`, { mode: 0o755 }); +} + +function extractedFiles(root: string): string[] { + const [archiveRoot] = fs.readdirSync(root); + const archiveRootPath = path.join(root, archiveRoot); + return fs.readdirSync(archiveRootPath).map((entry) => path.join(archiveRootPath, entry)); +} + +describe("compiled diagnostics CLI", () => { + it("creates a scoped redacted archive and rejects unknown sandboxes without partial output (#7617)", () => { + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-debug-contract-")); + const home = path.join(fixtureRoot, "home"); + const bin = path.join(fixtureRoot, "bin"); + const archive = path.join(fixtureRoot, "debug.tar.gz"); + const extractDir = path.join(fixtureRoot, "extracted"); + fs.mkdirSync(path.join(home, ".nemoclaw"), { recursive: true }); + fs.mkdirSync(bin, { recursive: true }); + fs.mkdirSync(extractDir, { recursive: true }); + fs.writeFileSync( + path.join(home, ".nemoclaw", "sandboxes.json"), + JSON.stringify({ + sandboxes: { + alpha: { + name: "alpha", + model: "test-model", + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + }, + }, + defaultSandbox: "alpha", + }), + { mode: 0o600 }, + ); + + writeExecutable(path.join(bin, "openshell"), [ + "#!/usr/bin/env bash", + 'case "$1 $2" in', + " \"sandbox list\") printf 'NAME\\nalpha Ready\\n'; exit 0 ;;", + ' "sandbox ssh-config") exit 1 ;;', + "esac", + `printf 'NVIDIA_INFERENCE_API_KEY=%s\\nGITHUB_TOKEN=%s\\n' ${NVIDIA_TEST_SECRET} ${GITHUB_TEST_SECRET}`, + ]); + writeExecutable(path.join(bin, "docker"), [ + "#!/usr/bin/env bash", + 'case "$*" in', + ' *"--format"*) exit 0 ;;', + "esac", + `printf 'container credential %s\\n' ${NVIDIA_TEST_SECRET}`, + ]); + for (const command of ["dmesg", "log", "nvidia-smi"]) { + writeExecutable(path.join(bin, command), [ + "#!/usr/bin/env bash", + `printf 'diagnostic credential %s\\n' ${GITHUB_TEST_SECRET}`, + ]); + } + + const env = { + ...process.env, + HOME: home, + PATH: `${bin}${path.delimiter}${process.env.PATH ?? ""}`, + }; + + try { + const result = runCli(["debug", "--quick", "--sandbox", "alpha", "--output", archive], env); + expect(result.status, result.output).toBe(0); + expect(result.output).toContain("Collecting diagnostics for sandbox 'alpha'"); + expect(result.output).not.toContain(NVIDIA_TEST_SECRET); + expect(result.output).not.toContain(GITHUB_TEST_SECRET); + expect(fs.statSync(archive).size).toBeGreaterThan(0); + + const extract = spawnSync("tar", ["xzf", archive, "-C", extractDir], { + encoding: "utf8", + timeout: 10_000, + }); + expect(extract.status, `${extract.stdout}${extract.stderr}`).toBe(0); + const archiveText = extractedFiles(extractDir) + .map((filePath) => fs.readFileSync(filePath, "utf8")) + .join("\n"); + expect(archiveText).toContain(""); + expect(archiveText).not.toContain(NVIDIA_TEST_SECRET); + expect(archiveText).not.toContain(GITHUB_TEST_SECRET); + expect(archiveText).not.toMatch(/nvapi-[A-Za-z0-9_-]{10,}|ghp_[A-Za-z0-9_-]{10,}/); + + const unknownArchive = path.join(fixtureRoot, "unknown.tar.gz"); + const unknown = runCli( + ["debug", "--quick", "--sandbox", "does-not-exist", "--output", unknownArchive], + env, + ); + expect(unknown.status).not.toBe(0); + expect(unknown.output).toContain("does-not-exist"); + expect(unknown.output).toContain("not registered"); + expect(fs.readdirSync(fixtureRoot).some((entry) => entry.startsWith("unknown.tar.gz"))).toBe( + false, + ); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/test/pr-e2e-gate-dispatch-recovery.test.ts b/test/pr-e2e-gate-dispatch-recovery.test.ts index 665eac9256..6eab27751a 100644 --- a/test/pr-e2e-gate-dispatch-recovery.test.ts +++ b/test/pr-e2e-gate-dispatch-recovery.test.ts @@ -797,7 +797,7 @@ describe("PR E2E dispatch-not-observed recovery", () => { await expect(startPrGate(startCommand(workDir))).resolves.toBeUndefined(); expect(requests.find((request) => request.url.endsWith("/dispatches"))?.body).toMatchObject({ inputs: { - jobs: "cloud-onboard,credential-sanitization,security-posture", + jobs: "cloud-inference,cloud-onboard,security-posture", checkout_sha: HEAD_SHA, base_sha: BASE_SHA, }, diff --git a/test/pr-e2e-gate-fork-approval.test.ts b/test/pr-e2e-gate-fork-approval.test.ts index fd24df30ac..c5364a1c4e 100644 --- a/test/pr-e2e-gate-fork-approval.test.ts +++ b/test/pr-e2e-gate-fork-approval.test.ts @@ -1017,7 +1017,7 @@ describe("PR E2E controller fork credentialed E2E approval safety", () => { expect(dispatch?.body).toMatchObject({ ref: "main", inputs: { - jobs: "cloud-onboard,credential-sanitization,security-posture,inference-routing,network-policy", + jobs: "cloud-inference,cloud-onboard,security-posture,inference-routing,network-policy", targets: "ubuntu-repo-cloud-langchain-deepagents-code", pr_number: "42", checkout_sha: HEAD_SHA, diff --git a/test/pr-e2e-gate-workflow.test.ts b/test/pr-e2e-gate-workflow.test.ts index 9c7b087afe..c3a1a73f31 100644 --- a/test/pr-e2e-gate-workflow.test.ts +++ b/test/pr-e2e-gate-workflow.test.ts @@ -702,7 +702,7 @@ describe("PR E2E gate workflow", () => { const retargeted = runChildValidation(HEAD_SHA, "d".repeat(40)); const racedWorkflow = runChildValidation(HEAD_SHA, BASE_SHA, "e".repeat(40)); const combined = runChildValidation(HEAD_SHA, BASE_SHA, WORKFLOW_SHA, { - jobs: "cloud-onboard,credential-sanitization,security-posture", + jobs: "cloud-inference,cloud-onboard,security-posture", targets: "ubuntu-repo-cloud-langchain-deepagents-code", }); const unapprovedTarget = runChildValidation(HEAD_SHA, BASE_SHA, WORKFLOW_SHA, { diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index 87f07d8904..0c0ea1a931 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -60,8 +60,8 @@ const BROAD_FILES = [ "src/lib/credentials/provider-list.ts", ] as const; const BROAD_JOBS = [ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", "channels-add-remove", "channels-stop-start", @@ -288,8 +288,8 @@ describe("PR E2E controller", () => { focusedPlan, ); expect(riskPlanRequiredJobIds(focusedPlan)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", "token-rotation", ]); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index c4733ba631..0f710bf3fe 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -28,7 +28,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); - expect(first.version).toBe(6); + expect(first.version).toBe(7); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -52,8 +52,8 @@ describe("deterministic PR risk plan", () => { expect(riskPlanRequiredJobIds(canonical)).toContain("cloud-onboard"); expect(ordinaryLiveTest.families.map((family) => family.id)).toEqual(["e2e-control-plane"]); expect(riskPlanRequiredJobIds(ordinaryLiveTest)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", ]); }); @@ -242,7 +242,7 @@ describe("deterministic PR risk plan", () => { { file: "src/lib/credentials/provider-list.ts", family: "credentials-security", - jobs: ["credential-sanitization", "security-posture"], + jobs: ["cloud-inference", "security-posture"], }, ])("maps $family changes to a reviewed E2E floor", ({ file, family, jobs }) => { const result = plan(file); @@ -268,22 +268,22 @@ describe("deterministic PR risk plan", () => { { file: "nemoclaw-blueprint/private-networks.yaml", families: ["inference-policy", "credentials-security"], - jobs: ["inference-routing", "network-policy", "credential-sanitization", "security-posture"], + jobs: ["inference-routing", "network-policy", "cloud-inference", "security-posture"], }, { file: "nemoclaw/src/blueprint/private-networks.ts", families: ["inference-policy", "credentials-security"], - jobs: ["inference-routing", "network-policy", "credential-sanitization", "security-posture"], + jobs: ["inference-routing", "network-policy", "cloud-inference", "security-posture"], }, { file: "src/lib/policy/managed-policy-binding.ts", families: ["inference-policy", "credentials-security"], - jobs: ["inference-routing", "network-policy", "credential-sanitization", "security-posture"], + jobs: ["inference-routing", "network-policy", "cloud-inference", "security-posture"], }, { file: "src/lib/shields/verify-lock.ts", families: ["credentials-security"], - jobs: ["credential-sanitization", "security-posture"], + jobs: ["cloud-inference", "security-posture"], }, ])("keeps the $file security boundary in the deterministic floor", ({ file, families, jobs }) => { const result = plan(file); @@ -327,7 +327,7 @@ describe("deterministic PR risk plan", () => { expect(result.families.map((family) => family.id)).toContain("e2e-control-plane"); expect(riskPlanRequiredJobIds(result)).toEqual( - expect.arrayContaining(["cloud-onboard", "credential-sanitization", "security-posture"]), + expect.arrayContaining(["cloud-onboard", "cloud-inference", "security-posture"]), ); }); @@ -389,8 +389,8 @@ describe("deterministic PR risk plan", () => { ); expect(riskPlanRequiredJobIds(result)).toEqual([ + "cloud-inference", "cloud-onboard", - "credential-sanitization", "security-posture", "channels-add-remove", "channels-stop-start", diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 6890e5ea8d..6adbad8c72 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; -export const RISK_PLAN_VERSION = 6 as const; +export const RISK_PLAN_VERSION = 7 as const; export const PR_E2E_TYPED_TARGET_IDS = ["ubuntu-repo-cloud-langchain-deepagents-code"] as const; @@ -267,7 +267,7 @@ export const RISK_RULES: readonly RiskRule[] = [ summary: "Credential and security-boundary changes must preserve secrecy, sanitization, and fail-closed policy behavior.", tier: 3, - requiredJobs: ["credential-sanitization", "security-posture"], + requiredJobs: ["cloud-inference", "security-posture"], invariants: [ "plaintext credentials do not cross logs, snapshots, artifacts, or sandbox boundaries", "invalid or missing security state fails closed", @@ -285,7 +285,7 @@ export const RISK_RULES: readonly RiskRule[] = [ summary: "E2E selection, execution, and evidence changes must preserve trusted dispatch and fail-closed result classification.", tier: 3, - requiredJobs: ["cloud-onboard", "credential-sanitization", "security-posture"], + requiredJobs: ["cloud-onboard", "cloud-inference", "security-posture"], invariants: [ "the controller selects only trusted jobs and binds results to the intended PR commit", "single-shard and matrix jobs both emit complete evidence through the canonical reporter", diff --git a/tools/e2e/brev-remote-vitest.mts b/tools/e2e/brev-remote-vitest.mts index 19b8c30851..81b5d428af 100644 --- a/tools/e2e/brev-remote-vitest.mts +++ b/tools/e2e/brev-remote-vitest.mts @@ -12,7 +12,6 @@ export const BREV_REMOTE_WRAPPER_GRACE_MS = 120_000; export const BREV_WORKFLOW_OWNERSHIP_ENV = "NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE"; const BREV_SUITES_WITHOUT_HARNESS_SANDBOX = new Set([ - "all", "full", "gpu", "messaging-compatible-endpoint", diff --git a/tools/e2e/retired-selector-compatibility.mts b/tools/e2e/retired-selector-compatibility.mts index 96967ea0e9..26042ce84c 100644 --- a/tools/e2e/retired-selector-compatibility.mts +++ b/tools/e2e/retired-selector-compatibility.mts @@ -29,7 +29,7 @@ const SELECTOR_LIST_PATTERN = /^[A-Za-z0-9_-]+(?:,[A-Za-z0-9_-]+)*$/u; type RetiredControllerSelectorId = (typeof RETIRED_CONTROLLER_SELECTOR_IDS)[number]; type ReplacementTest = { files: readonly string[]; - project: "integration" | "installer-integration" | "package-contract"; + project: "cli" | "integration" | "installer-integration" | "package-contract"; }; type Replacement = { legacyFile?: string; @@ -38,6 +38,33 @@ type Replacement = { const RETIRED_SELECTOR_ID_SET = new Set(RETIRED_CONTROLLER_SELECTOR_IDS); const REPLACEMENTS: Readonly> = { + "credential-migration": { + legacyFile: "test/e2e/live/credential-migration.test.ts", + tests: [ + { + files: ["test/credential-migration-reconciliation.test.ts"], + project: "integration", + }, + ], + }, + "credential-sanitization": { + legacyFile: "test/e2e/live/credential-sanitization.test.ts", + tests: [ + { + files: ["src/lib/security/credential-filter-secret-patterns.test.ts"], + project: "cli", + }, + ], + }, + diagnostics: { + legacyFile: "test/e2e/live/diagnostics.test.ts", + tests: [ + { + files: ["test/package-contract/cli/debug-cli-command.test.ts"], + project: "package-contract", + }, + ], + }, "docs-validation": { legacyFile: "test/e2e/live/docs-validation.test.ts", tests: [ @@ -125,7 +152,12 @@ function replacementCommands(selected: readonly RetiredControllerSelectorId[]): filesByProject.set(test.project, files); } } - for (const project of ["integration", "installer-integration", "package-contract"] as const) { + for (const project of [ + "cli", + "integration", + "installer-integration", + "package-contract", + ] as const) { const files = filesByProject.get(project); if (!files) continue; commands.push({ diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 8bdfc2355b..05d6ec38f5 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -122,6 +122,9 @@ type CachedFreeStandingJobsInventory = { const SELECTOR_PATTERN = /^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$/; const SELECTOR_ID_PATTERN = /^[A-Za-z0-9_-]+$/; export const RETIRED_CONTROLLER_SELECTOR_IDS = [ + "credential-migration", + "credential-sanitization", + "diagnostics", "docs-validation", "gateway-drift-preflight", "gateway-health-honest", @@ -2862,90 +2865,6 @@ function validateHermesTimeoutHeadroom(errors: string[], jobs: WorkflowRecord): } } -function validateDiagnosticsJob(errors: string[], jobs: WorkflowRecord): void { - const jobName = "diagnostics"; - const targetName = "diagnostics"; - const job = asRecord(jobs[jobName]); - if (Object.keys(job).length === 0) { - errors.push("workflow missing diagnostics job"); - return; - } - - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("diagnostics job must run on ubuntu-latest"); - } - validateFreeStandingJobSelector(errors, jobs, jobName, targetName); - if (job["timeout-minutes"] !== 60) { - errors.push("diagnostics job must keep the 60 minute timeout"); - } - - const jobEnv = asRecord(job.env); - if ("DOCKER_CONFIG" in jobEnv) { - errors.push("diagnostics job must not expose Docker auth to branch-controlled steps"); - } - if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/live/diagnostics") { - errors.push("diagnostics job must write artifacts under e2e-artifacts/live/diagnostics"); - } - if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { - errors.push("diagnostics job must point NEMOCLAW_CLI_BIN at the repo CLI"); - } - if (jobEnv.NEMOCLAW_RUN_LIVE_E2E !== "1") { - errors.push("diagnostics job must set NEMOCLAW_RUN_LIVE_E2E=1"); - } - if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { - errors.push("diagnostics job must set NEMOCLAW_NON_INTERACTIVE=1"); - } - if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { - errors.push("diagnostics job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); - } - if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-diag") { - errors.push("diagnostics job must use the stable e2e-diag sandbox name"); - } - if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { - errors.push("diagnostics job must force OPENSHELL_GATEWAY=nemoclaw"); - } - for (const secret of [ - "NVIDIA_INFERENCE_API_KEY", - "DOCKERHUB_USERNAME", - "DOCKERHUB_TOKEN", - "GITHUB_TOKEN", - ]) { - requireEnvDoesNotExposeSecret(errors, "diagnostics job", jobEnv, secret); - } - - const steps = asSteps(job.steps); - requireNoDispatchInputInterpolation(errors, steps); - for (const step of steps) { - const stepName = `diagnostics step '${step.name ?? step.uses ?? ""}'`; - const stepEnv = asRecord(step.env); - if (step.name !== "Run diagnostics live test") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); - } - if (step.name !== DOCKER_HUB_AUTH_STEP) { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); - requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); - } - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); - } - - const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); - if (!checkout) errors.push("diagnostics job missing checkout step"); - requireFullShaAction(errors, checkout, "diagnostics checkout"); - if (asRecord(checkout?.with)["persist-credentials"] !== false) { - errors.push("diagnostics checkout step must set persist-credentials=false"); - } - - const runVitest = requireJobStep(errors, jobName, steps, "Run diagnostics live test"); - const runVitestEnv = asRecord(runVitest?.env); - if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { - errors.push("diagnostics live E2E step must receive NVIDIA_INFERENCE_API_KEY from secrets"); - } - requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); - requireRunContains(errors, runVitest, "test/e2e/live/diagnostics.test.ts"); - requireRunDoesNotContain(errors, runVitest, "${{ inputs."); -} - function validateSparkInstallJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "spark-install"; const targetName = "spark-install"; @@ -4855,7 +4774,6 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateStagingBrevLaunchableJob(errors, jobs); validateStagingBrevLaunchableReadinessJob(errors, jobs); validateSkillAgentJob(errors, jobs); - validateFreeStandingJobSelector(errors, jobs, "credential-migration", "credential-migration"); validateFreeStandingJobSelector(errors, jobs, "sessions-agents-cli", "sessions-agents-cli"); validateFreeStandingJobSelector(errors, jobs, "inference-routing", "inference-routing"); validateInferenceRoutingJob(errors, jobs); @@ -4885,7 +4803,6 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { ); validateIssue4434HostDependencies(errors, jobs); validateOpenclawTuiChatCorrelationHostDependencies(errors, jobs); - validateDiagnosticsJob(errors, jobs); validateModelRouterProviderRoutedInferenceJob(errors, jobs); validateSnapshotCommandsJob(errors, jobs); errors.push(...validateSandboxOperationsWorkflow({ jobs }));