diff --git a/scripts/checks/e2e-mock-parity.mts b/scripts/checks/e2e-mock-parity.mts index 93bcbc23fc2..847cede0c5d 100644 --- a/scripts/checks/e2e-mock-parity.mts +++ b/scripts/checks/e2e-mock-parity.mts @@ -8,11 +8,14 @@ import { fileURLToPath } from "node:url"; import ts from "typescript"; +import { moduleTagDeclarations } from "../../tools/e2e/module-tags.mts"; + const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); export const DEFAULT_PARITY_MANIFEST = "test/e2e/mock-parity.json"; export type MockParityEntry = { live: string; + liveSources?: string[]; fast?: string[]; liveOnlyReason?: string; }; @@ -23,6 +26,7 @@ export type MockParityManifest = { }; const LIVE_TEST = /^test\/e2e\/live\/.+\.test\.ts$/u; +const LIVE_HELPER = /^test\/e2e\/live\/(?!.*\.test\.ts$).+\.ts$/u; const FAST_TESTS = [ /^src\/.+\.test\.ts$/u, /^nemoclaw\/src\/.+\.test\.ts$/u, @@ -50,7 +54,10 @@ function sourceTokens(source: string): string { for (const child of children) visit(child); }; visit(sourceFile); - return JSON.stringify(tokens); + return JSON.stringify({ + moduleTags: moduleTagDeclarations(source).map(({ tag }) => tag), + tokens, + }); } export function isMockParityRelevantSourceChange( @@ -91,6 +98,7 @@ export function validateMockParity(options: { } const entries = new Map(); + const sourceOwners = new Map(); for (const entry of manifest.entries) { if (!entry || typeof entry !== "object" || typeof entry.live !== "string") { errors.push("mock parity entries must be objects with a live path"); @@ -106,6 +114,14 @@ export function validateMockParity(options: { } entries.set(entry.live, entry); + if ( + entry.liveSources !== undefined && + (!Array.isArray(entry.liveSources) || + entry.liveSources.some((file) => typeof file !== "string")) + ) { + errors.push(`${entry.live}: liveSources must be an array of live E2E helper paths`); + continue; + } if ( entry.fast !== undefined && (!Array.isArray(entry.fast) || entry.fast.some((file) => typeof file !== "string")) @@ -126,6 +142,18 @@ export function validateMockParity(options: { } if (!fileExists(entry.live)) errors.push(`${entry.live}: live test does not exist`); + for (const sourceFile of new Set(entry.liveSources ?? [])) { + if (!isSafeRepoPath(sourceFile) || !LIVE_HELPER.test(sourceFile)) { + errors.push(`${entry.live}: ${sourceFile} is not a test/e2e/live/**/*.ts helper file`); + continue; + } + if (!fileExists(sourceFile)) { + errors.push(`${entry.live}: live E2E helper does not exist: ${sourceFile}`); + } + const owners = sourceOwners.get(sourceFile) ?? []; + owners.push(entry); + sourceOwners.set(sourceFile, owners); + } for (const fastFile of new Set(fast)) { if (!isFastPrTest(fastFile)) { errors.push(`${entry.live}: ${fastFile} is not collected by a fast PR test project`); @@ -135,10 +163,41 @@ export function validateMockParity(options: { } } - for (const liveFile of [...new Set(changedFiles)].filter((file) => LIVE_TEST.test(file))) { - if (!entries.has(liveFile)) { + const changedFileSet = new Set(changedFiles); + const requireChangedFastTest = (entry: MockParityEntry, changedSource: string): void => { + const mappedFastTests = Array.isArray(entry.fast) + ? entry.fast.filter((fastFile): fastFile is string => typeof fastFile === "string") + : []; + if ( + mappedFastTests.length > 0 && + !mappedFastTests.some((fastFile) => changedFileSet.has(fastFile)) + ) { + errors.push( + changedSource === entry.live + ? `${entry.live}: change at least one mapped fast PR test with the live E2E` + : `${changedSource}: change at least one fast PR test mapped from ${entry.live}`, + ); + } + }; + + for (const liveFile of [...changedFileSet].filter((file) => LIVE_TEST.test(file))) { + const entry = entries.get(liveFile); + if (!entry) { errors.push(`${liveFile}: changed live E2E needs an entry in ${DEFAULT_PARITY_MANIFEST}`); + continue; + } + requireChangedFastTest(entry, liveFile); + } + + for (const helperFile of [...changedFileSet].filter((file) => LIVE_HELPER.test(file))) { + const owners = sourceOwners.get(helperFile) ?? []; + if (owners.length === 0) { + errors.push( + `${helperFile}: changed live E2E helper needs an owning entry in ${DEFAULT_PARITY_MANIFEST}`, + ); + continue; } + for (const owner of owners) requireChangedFastTest(owner, helperFile); } return errors.sort(); @@ -161,6 +220,18 @@ function sourceAtRef(ref: string, file: string): string | null { } } +/** Remove metadata-only live and fast test changes before parity validation. */ +export function filterMockParityRelevantChangedFiles( + files: readonly string[], + sourceAtBase: (file: string) => string | null, + sourceAtHead: (file: string) => string | null, +): string[] { + return files.filter((file) => { + if (!LIVE_TEST.test(file) && !LIVE_HELPER.test(file) && !isFastPrTest(file)) return true; + return isMockParityRelevantSourceChange(sourceAtBase(file), sourceAtHead(file)); + }); +} + function changedFiles(base: string, head: string): string[] { const files = execFileSync( "git", @@ -172,10 +243,10 @@ function changedFiles(base: string, head: string): string[] { ) .split(/\r?\n/u) .filter(Boolean); - return files.filter( - (file) => - !LIVE_TEST.test(file) || - isMockParityRelevantSourceChange(sourceAtRef(base, file), sourceAtRef(head, file)), + return filterMockParityRelevantChangedFiles( + files, + (file) => sourceAtRef(base, file), + (file) => sourceAtRef(head, file), ); } diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index 5edab936a48..5c4cd667768 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -151,7 +151,7 @@ function withSandboxMutationLockUnlessPreview( * Internal composition dependencies for channel mutation. * * The Google Chat capability is intentionally absent from the public CLI - * composition. The live E2E entrypoint supplies it directly so environment + * composition. The channels stop/start live E2E helper supplies it directly so environment * variables and predictable sandbox names cannot enable non-interactive * audience enrollment in ordinary production execution. */ diff --git a/test/automation/e2e/e2e-mock-parity.test.ts b/test/automation/e2e/e2e-mock-parity.test.ts index a97a41b3afc..ccd0c346746 100644 --- a/test/automation/e2e/e2e-mock-parity.test.ts +++ b/test/automation/e2e/e2e-mock-parity.test.ts @@ -1,41 +1,38 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { type SpawnSyncReturns, spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + filterMockParityRelevantChangedFiles, isMockParityRelevantSourceChange, type MockParityManifest, validateMockParity, } from "../../../scripts/checks/e2e-mock-parity.mts"; -import { type CompositeAction, readYaml } from "../../helpers/e2e-workflow-contract"; const live = "test/e2e/live/example.test.ts"; +const liveHelper = "test/e2e/live/example-helper.ts"; const fast = "test/e2e/support/example.test.ts"; const TAGGED_NEW_SOURCE = "// @module-tag e2e/credential-free\n"; -const exists = (file: string) => file === live || file === fast; +const exists = (file: string) => file === live || file === liveHelper || file === fast; function manifest(entries: MockParityManifest["entries"]): MockParityManifest { return { version: 1, entries }; } describe("changed live E2E mock parity", () => { - it("treats module-tag-only diffs as metadata", () => { + it("retains recognized module-tag changes while ignoring ordinary comments", () => { expect( isMockParityRelevantSourceChange( "// SPDX-License-Identifier: Apache-2.0\n\nexport {};\n", "// SPDX-License-Identifier: Apache-2.0\n// @module-tag e2e/credential-free\n\nexport {};\n", ), - ).toBe(false); + ).toBe(true); expect( isMockParityRelevantSourceChange( - `${"// @module"}-tag retired/value\n\nexport {};\n`, "// @module-tag e2e/credential-free\n\nexport {};\n", + "export {};\n", ), - ).toBe(false); + ).toBe(true); expect( isMockParityRelevantSourceChange( "// old terminology\nexport {};\n", @@ -59,21 +56,149 @@ describe("changed live E2E mock parity", () => { "// SPDX-License-Identifier: Apache-2.0\n\nexport {};\n", "// SPDX-License-Identifier: Apache-2.0\n/* @module-tag e2e/credential-free */\n\nexport {};\n", ), + ).toBe(true); + expect( + isMockParityRelevantSourceChange( + "/* @module-tag e2e/credential-free */\n\nexport {};\n", + "export {};\n", + ), + ).toBe(true); + expect( + isMockParityRelevantSourceChange( + `${"// @module"}-tag retired.value\n\nexport {};\n`, + "// another ordinary comment\n\nexport {};\n", + ), ).toBe(false); expect(isMockParityRelevantSourceChange(null, null)).toBe(true); expect(isMockParityRelevantSourceChange(null, TAGGED_NEW_SOURCE)).toBe(true); }); + it.each([ + { + baseLive: "export const liveBehavior = 1;\n", + headLive: "// @module-tag e2e/credential-free\nexport const liveBehavior = 1;\n", + title: "adding a recognized module tag", + }, + { + baseLive: "// @module-tag e2e/credential-free\nexport const liveBehavior = 1;\n", + headLive: "export const liveBehavior = 1;\n", + title: "removing a recognized module tag", + }, + ])("requires mapped fast coverage after $title", ({ baseLive, headLive }) => { + const relevantFiles = filterMockParityRelevantChangedFiles( + [live, fast], + (file) => (file === live ? baseLive : "export const fastBehavior = 1;\n"), + (file) => + file === live ? headLive : "// ordinary comment\nexport const fastBehavior = 1;\n", + ); + + expect(relevantFiles).toEqual([live]); + expect( + validateMockParity({ + manifest: manifest([{ live, fast: [fast] }]), + changedFiles: relevantFiles, + fileExists: exists, + }), + ).toEqual([`${live}: change at least one mapped fast PR test with the live E2E`]); + }); + it("accepts a changed live E2E mapped to a fast PR test", () => { + expect( + validateMockParity({ + manifest: manifest([{ live, fast: [fast] }]), + changedFiles: [live, fast], + fileExists: exists, + }), + ).toEqual([]); + }); + + it("rejects a stale fast-test mapping for changed live behavior", () => { expect( validateMockParity({ manifest: manifest([{ live, fast: [fast] }]), changedFiles: [live], fileExists: exists, }), + ).toEqual([`${live}: change at least one mapped fast PR test with the live E2E`]); + }); + + it("requires mapped fast coverage when a declared live E2E helper changes", () => { + expect( + validateMockParity({ + manifest: manifest([{ live, liveSources: [liveHelper], fast: [fast] }]), + changedFiles: [liveHelper], + fileExists: exists, + }), + ).toEqual([`${liveHelper}: change at least one fast PR test mapped from ${live}`]); + }); + + it("accepts a changed live E2E helper with a changed mapped fast test", () => { + expect( + validateMockParity({ + manifest: manifest([{ live, liveSources: [liveHelper], fast: [fast] }]), + changedFiles: [liveHelper, fast], + fileExists: exists, + }), ).toEqual([]); }); + it("rejects a changed live E2E helper without an owning manifest entry", () => { + expect( + validateMockParity({ + manifest: manifest([{ live, fast: [fast] }]), + changedFiles: [liveHelper, fast], + fileExists: exists, + }), + ).toEqual([ + `${liveHelper}: changed live E2E helper needs an owning entry in test/e2e/mock-parity.json`, + ]); + }); + + it("filters a comment-only live E2E helper change before ownership validation", () => { + const relevantFiles = filterMockParityRelevantChangedFiles( + [liveHelper], + () => "// old wording\nexport const helper = true;\n", + () => "// current wording\nexport const helper = true;\n", + ); + + expect(relevantFiles).toEqual([]); + }); + + it.each([ + { + expected: [], + fastHead: "export const fastBehavior = 2;\n", + title: "retains a token-changing mapped fast test", + }, + { + expected: [`${live}: change at least one mapped fast PR test with the live E2E`], + fastHead: "// formatting only\n\nexport const fastBehavior = 1;\n", + title: "filters a comment-and-whitespace-only mapped fast test", + }, + ])("$title", ({ expected, fastHead }) => { + const baseSources = new Map([ + [live, "export const liveBehavior = 1;\n"], + [fast, "export const fastBehavior = 1;\n"], + ]); + const headSources = new Map([ + [live, "export const liveBehavior = 2;\n"], + [fast, fastHead], + ]); + const relevantFiles = filterMockParityRelevantChangedFiles( + [live, fast], + (file) => baseSources.get(file) ?? null, + (file) => headSources.get(file) ?? null, + ); + + expect( + validateMockParity({ + manifest: manifest([{ live, fast: [fast] }]), + changedFiles: relevantFiles, + fileExists: exists, + }), + ).toEqual(expected); + }); + it("rejects a changed live E2E without a parity decision", () => { expect( validateMockParity({ manifest: manifest([]), changedFiles: [live], fileExists: exists }), @@ -84,7 +209,7 @@ describe("changed live E2E mock parity", () => { expect( validateMockParity({ manifest: manifest([{ live, fast: ["test/e2e/live/not-fast.test.ts", fast] }]), - changedFiles: [live], + changedFiles: [live, fast], fileExists: (file) => file === live, }), ).toEqual([ @@ -113,80 +238,3 @@ describe("changed live E2E mock parity", () => { ).toEqual([`${live}: liveOnlyReason must be a string`]); }); }); - -describe("trusted E2E parity entrypoint selection", () => { - const action = readYaml(".github/actions/ci-cli-coverage-shard/action.yaml"); - const parityStep = action.runs.steps.find( - (step) => step.name === "Validate changed live E2E mock parity", - ); - const parityRun = parityStep?.run ?? ""; - const parityEnv = parityStep?.env ?? {}; - const stem = "scripts/checks/e2e-mock-parity"; - - function withParityEntrypoints( - extensions: readonly string[], - verify: (result: SpawnSyncReturns, commandLog: string) => void, - ): void { - const temp = mkdtempSync(join(tmpdir(), "nemoclaw-e2e-parity-entrypoint-")); - const fakeBin = join(temp, "bin"); - const commandLog = join(temp, "command.json"); - mkdirSync(fakeBin); - mkdirSync(join(temp, "scripts", "checks"), { recursive: true }); - writeFileSync( - join(fakeBin, "npx"), - [ - "#!/usr/bin/env node", - 'const fs = require("node:fs");', - "fs.appendFileSync(process.env.COMMAND_LOG, `${JSON.stringify(process.argv.slice(2))}\\n`);", - ].join("\n"), - { mode: 0o755 }, - ); - for (const extension of extensions) { - writeFileSync(join(temp, `${stem}.${extension}`), "// fixture\n"); - } - - try { - verify( - spawnSync("bash", ["-c", parityRun], { - cwd: temp, - encoding: "utf8", - env: { - ...process.env, - ...parityEnv, - COMMAND_LOG: commandLog, - EVENT_NAME: "pull_request", - PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - PUSH_BASE_SHA: "unused", - }, - timeout: 5_000, - }), - commandLog, - ); - } finally { - rmSync(temp, { force: true, recursive: true }); - } - } - - it("runs the migrated .mts entrypoint (#6918)", () => { - withParityEntrypoints(["mts"], (result, commandLog) => { - expect(result.status, String(result.stderr)).toBe(0); - expect( - readFileSync(commandLog, "utf8") - .trim() - .split("\n") - .map((line) => JSON.parse(line)), - ).toEqual([["tsx", `${stem}.mts`, "--base", "HEAD^1", "--head", "HEAD^2"]]); - }); - }); - - it.each([ - { extensions: [], title: "rejects a missing parity entrypoint" }, - { extensions: ["ts"], title: "rejects the retired .ts parity entrypoint" }, - ])("$title (#6918)", ({ extensions }) => { - withParityEntrypoints(extensions, (result, commandLog) => { - expect(result.status, String(result.stderr)).toBe(1); - expect(existsSync(commandLog)).toBe(false); - expect(String(result.stdout)).toContain("Missing E2E mock parity entrypoint"); - }); - }); -}); diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 2fca7b64407..bc488d0e207 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -249,12 +249,15 @@ command execution, test outcomes, or registered resource release. The retired `--emit-matrix` and `--plan-only` paths must not be reintroduced. -When adding or changing a live test, update `test/e2e/mock-parity.json` with -the fast PR-collected test that covers its mockable contract. If the behavior -cannot be reproduced without real infrastructure, record a concise -`liveOnlyReason` instead. The PR and `main` CLI coverage shards enforce this -changed-file policy alongside the `e2e-support` project without requiring an -immediate backfill of untouched tests. +When you add or make a non-comment source change to a live E2E test or a +`test/e2e/live/` helper, update `test/e2e/mock-parity.json`. List each changed +helper under `liveSources` for its owning live test. If the entry has mapped +fast tests, make a non-comment source change to at least one mapped fast test +in the same PR. Use +`liveOnlyReason` only when no fast test can reproduce the contract. The PR and +`main` CLI coverage shards enforce this changed-file policy alongside the +`e2e-support` project without requiring an immediate backfill of untouched +tests. ## Repository Layout diff --git a/test/e2e/live/channels-stop-start-googlechat-entry.ts b/test/e2e/live/channels-stop-start-googlechat-entry.ts deleted file mode 100644 index dd3d9b23a97..00000000000 --- a/test/e2e/live/channels-stop-start-googlechat-entry.ts +++ /dev/null @@ -1,301 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -import type { AddSandboxChannelDependencies } from "../../../src/lib/actions/sandbox/policy-channel.ts"; -import * as policyChannelDependenciesModule from "../../../src/lib/actions/sandbox/policy-channel-dependencies.ts"; -import * as policyChannelModule from "../../../src/lib/actions/sandbox/policy-channel.ts"; -import * as openshellRuntimeModule from "../../../src/lib/adapters/openshell/runtime.ts"; -import * as messagingBridgeProviderModule from "../../../src/lib/onboard/messaging-bridge-provider.ts"; -import * as onboardProvidersModule from "../../../src/lib/onboard/providers.ts"; -import * as statePathsModule from "../../../src/lib/state/paths.ts"; -import { assertChannelsStopStartSandboxName } from "./channels-stop-start-safety.ts"; -import type { AgentKind } from "./phase6-messaging-helpers.ts"; - -type PolicyChannelModule = typeof import("../../../src/lib/actions/sandbox/policy-channel.ts"); -type PolicyChannelDependenciesModule = - typeof import("../../../src/lib/actions/sandbox/policy-channel-dependencies.ts"); -type OpenshellRuntimeModule = typeof import("../../../src/lib/adapters/openshell/runtime.ts"); -type MessagingBridgeProviderModule = - typeof import("../../../src/lib/onboard/messaging-bridge-provider.ts"); -type StatePathsModule = typeof import("../../../src/lib/state/paths.ts"); -type ProviderUpsertOptions = { - readonly replaceExisting?: boolean; - readonly revalidatePolicyRequirements?: (operation: string) => void; -}; -type ProviderDependencies = { - upsertMessagingProviders( - tokenDefs: Parameters[0], - run: typeof runOpenshell, - options?: ProviderUpsertOptions, - ): string[]; -}; - -const policyChannel = ( - "default" in policyChannelModule ? policyChannelModule.default : policyChannelModule -) as PolicyChannelModule; -const { addSandboxChannel } = policyChannel; -const policyChannelDependenciesNamespace = ( - "default" in policyChannelDependenciesModule - ? policyChannelDependenciesModule.default - : policyChannelDependenciesModule -) as PolicyChannelDependenciesModule; -const { policyChannelDependencies } = policyChannelDependenciesNamespace; -const openshellRuntime = ( - "default" in openshellRuntimeModule ? openshellRuntimeModule.default : openshellRuntimeModule -) as OpenshellRuntimeModule; -const { runOpenshell } = openshellRuntime; -const messagingBridgeProvider = ( - "default" in messagingBridgeProviderModule - ? messagingBridgeProviderModule.default - : messagingBridgeProviderModule -) as MessagingBridgeProviderModule; -const { ensureMessagingBridgeProfiles } = messagingBridgeProvider; -const onboardProviders = ( - "default" in onboardProvidersModule ? onboardProvidersModule.default : onboardProvidersModule -) as ProviderDependencies; -const statePaths = ( - "default" in statePathsModule ? statePathsModule.default : statePathsModule -) as StatePathsModule; -const { ROOT } = statePaths; - -interface GooglechatLiveE2eComposition { - readonly sandboxName: string; - readonly agent: AgentKind; - readonly audience: string; -} - -interface GooglechatLiveE2eDependencies { - readonly addSandboxChannel: ( - sandboxName: string, - options: { readonly channel: string }, - dependencies: AddSandboxChannelDependencies, - ) => Promise; - readonly installCredentialFixture: (sandboxName: string, agent: AgentKind) => () => void; - readonly rebuildSandbox?: (sandboxName: string, args: string[]) => Promise; -} - -interface GooglechatCredentialFixtureDependencies { - readonly ensureProfiles?: typeof ensureMessagingBridgeProfiles; - readonly providerDependencies?: ProviderDependencies; - readonly root?: string; - readonly run?: typeof runOpenshell; -} - -export const GOOGLECHAT_E2E_ACCESS_TOKEN = "e2e-fake-googlechat-access-token"; - -const PROVIDER_TYPE_BY_AGENT: Readonly> = { - openclaw: "google-chat-bridge", - hermes: "google-chat-hermes-bridge", -}; - -/** - * Replace Google Chat's asynchronous Google OAuth mint only inside this live-test - * composition root. The fixed value is not a credential. Creating the real - * OpenShell provider with it still exercises provider identity, revision-scoped - * sandbox injection, endpoint binding, L7 rewrite, and removal without requiring - * a Google service account in CI. - */ -export function installGooglechatCredentialFixture( - sandboxName: string, - agent: AgentKind, - dependencies: GooglechatCredentialFixtureDependencies = {}, -): () => void { - assertChannelsStopStartSandboxName(sandboxName, agent); - const ensureProfiles = dependencies.ensureProfiles ?? ensureMessagingBridgeProfiles; - const providerDependencies = dependencies.providerDependencies ?? onboardProviders; - const root = dependencies.root ?? ROOT; - const run = dependencies.run ?? runOpenshell; - const expectedName = `${sandboxName}-googlechat-bridge`; - const expectedType = PROVIDER_TYPE_BY_AGENT[agent]; - const original = providerDependencies.upsertMessagingProviders; - - providerDependencies.upsertMessagingProviders = (tokenDefs, providerRun, options = {}) => { - const fixtureTokenDefs = tokenDefs.filter(({ name }) => name === expectedName); - const fixtureTokenDef = fixtureTokenDefs[0]; - if ( - fixtureTokenDefs.length !== 1 || - fixtureTokenDef?.envKey !== "GOOGLE_CHAT_ACCESS_TOKEN" || - fixtureTokenDef?.providerType !== expectedType - ) { - throw new Error("Google Chat live fixture received an unexpected provider definition"); - } - - const delegatedTokenDefs = tokenDefs.filter(({ name }) => name !== expectedName); - const delegatedProviderNames = - delegatedTokenDefs.length === 0 ? [] : original(delegatedTokenDefs, providerRun, options); - const baseRun = providerRun ?? run; - const revalidate = () => - options.revalidatePolicyRequirements?.( - `manage Google Chat live fixture provider '${expectedName}'`, - ); - const effectiveRun: typeof runOpenshell = (args, runOptions) => { - revalidate(); - return baseRun(args, runOptions); - }; - ensureProfiles(fixtureTokenDefs, { - root, - runOpenshell: effectiveRun, - redact: (value) => value.replaceAll(GOOGLECHAT_E2E_ACCESS_TOKEN, "[redacted]"), - }); - const existing = effectiveRun(["provider", "get", expectedName], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); - if (existing.status === 0 && options.replaceExisting) { - const removed = effectiveRun(["provider", "delete", expectedName], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); - if (removed.status !== 0) { - throw new Error(`Google Chat live fixture could not replace provider '${expectedName}'`); - } - } - const action = existing.status === 0 && !options.replaceExisting ? "update" : "create"; - const providerArgs = - action === "update" - ? ["provider", "update", expectedName, "--credential", "GOOGLE_CHAT_ACCESS_TOKEN"] - : [ - "provider", - "create", - "--name", - expectedName, - "--type", - expectedType, - "--credential", - "GOOGLE_CHAT_ACCESS_TOKEN", - ]; - const mutated = effectiveRun(providerArgs, { - env: { GOOGLE_CHAT_ACCESS_TOKEN: GOOGLECHAT_E2E_ACCESS_TOKEN }, - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); - if (mutated.status !== 0) { - throw new Error(`Google Chat live fixture could not ${action} provider '${expectedName}'`); - } - const registered = new Set([...delegatedProviderNames, expectedName]); - return tokenDefs.map(({ name }) => name).filter((name) => registered.has(name)); - }; - - return () => { - providerDependencies.upsertMessagingProviders = original; - }; -} - -const DEFAULT_DEPENDENCIES: GooglechatLiveE2eDependencies = { - addSandboxChannel, - installCredentialFixture: installGooglechatCredentialFixture, - rebuildSandbox: (sandboxName, args) => - policyChannelDependencies.rebuildSandbox(sandboxName, args), -}; - -function requireLiveAudience(input: GooglechatLiveE2eComposition): string { - assertChannelsStopStartSandboxName(input.sandboxName, input.agent); - const audience = input.audience.trim(); - if (!audience) { - throw new Error("GOOGLECHAT_AUDIENCE is required for the channels-stop-start live target"); - } - return audience; -} - -async function addGooglechatWithInstalledFixture( - input: GooglechatLiveE2eComposition, - audience: string, - dependencies: GooglechatLiveE2eDependencies, -): Promise { - await dependencies.addSandboxChannel( - input.sandboxName, - { channel: "googlechat" }, - input.agent === "openclaw" - ? { - googlechatNonInteractiveAudienceCapability: Object.freeze({ - audience, - }), - } - : {}, - ); -} - -/** - * The sole composition root that grants non-interactive Google Chat audience - * enrollment. Production CLI composition does not receive this capability. - */ -export async function addGooglechatForChannelsStopStartLiveE2e( - input: GooglechatLiveE2eComposition, - dependencies: GooglechatLiveE2eDependencies = DEFAULT_DEPENDENCIES, -): Promise { - const audience = requireLiveAudience(input); - - const restore = dependencies.installCredentialFixture(input.sandboxName, input.agent); - try { - await addGooglechatWithInstalledFixture(input, audience, dependencies); - } finally { - restore(); - } -} - -/** Keep the fake OAuth mint installed across both provider registrations. */ -export async function addAndRebuildGooglechatForChannelsStopStartLiveE2e( - input: GooglechatLiveE2eComposition, - dependencies: GooglechatLiveE2eDependencies = DEFAULT_DEPENDENCIES, -): Promise { - const audience = requireLiveAudience(input); - if (!dependencies.rebuildSandbox) { - throw new Error("Google Chat live rebuild dependency is unavailable"); - } - - const restore = dependencies.installCredentialFixture(input.sandboxName, input.agent); - try { - await addGooglechatWithInstalledFixture(input, audience, dependencies); - await dependencies.rebuildSandbox(input.sandboxName, ["--yes"]); - } finally { - restore(); - } -} - -/** Keep the fake OAuth mint installed while a later lifecycle rebuild reconciles Google Chat. */ -export async function rebuildGooglechatForChannelsStopStartLiveE2e( - input: Pick, - dependencies: GooglechatLiveE2eDependencies = DEFAULT_DEPENDENCIES, -): Promise { - if (!dependencies.rebuildSandbox) { - throw new Error("Google Chat live rebuild dependency is unavailable"); - } - - const restore = dependencies.installCredentialFixture(input.sandboxName, input.agent); - try { - await dependencies.rebuildSandbox(input.sandboxName, ["--yes"]); - } finally { - restore(); - } -} - -async function main(): Promise { - const agent = (process.env.NEMOCLAW_CHANNELS_STOP_START_AGENT ?? process.env.NEMOCLAW_AGENT) as - | AgentKind - | undefined; - if (agent !== "openclaw" && agent !== "hermes") { - throw new Error("NEMOCLAW_CHANNELS_STOP_START_AGENT must be openclaw or hermes"); - } - const sandboxName = process.argv[2] ?? ""; - const mode = process.argv[3]; - if (mode === "--rebuild-only") { - await rebuildGooglechatForChannelsStopStartLiveE2e({ sandboxName, agent }); - return; - } - if (mode) throw new Error(`unknown Google Chat live E2E mode '${mode}'`); - await addAndRebuildGooglechatForChannelsStopStartLiveE2e({ - sandboxName, - agent, - audience: process.env.GOOGLECHAT_AUDIENCE ?? "", - }); -} - -if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1] ?? "")) { - main().catch((error: unknown) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; - }); -} diff --git a/test/e2e/live/channels-stop-start-googlechat-proof.ts b/test/e2e/live/channels-stop-start-googlechat-proof.ts index 0dcb0f91cae..7aba97050b2 100644 --- a/test/e2e/live/channels-stop-start-googlechat-proof.ts +++ b/test/e2e/live/channels-stop-start-googlechat-proof.ts @@ -88,18 +88,37 @@ print(json.dumps({ })) `; -interface GooglechatBoundaryProof { +export interface GooglechatProviderEgressProof { readonly installedOverride?: boolean; readonly placeholder: string; readonly statuses: number[]; } -function parseBoundaryProof(stdout: string): GooglechatBoundaryProof { +export function parseGooglechatProviderEgressProof(stdout: string): GooglechatProviderEgressProof { const lastLine = stdout.trim().split(/\r?\n/u).at(-1) ?? ""; - return JSON.parse(lastLine) as GooglechatBoundaryProof; + return JSON.parse(lastLine) as GooglechatProviderEgressProof; } -export async function expectGooglechatCredentialBoundary( +export function assertGooglechatProviderEgressProof( + proof: GooglechatProviderEgressProof, + agent: AgentKind, +): void { + expect(proof.placeholder, "sandbox process received a revision-scoped placeholder").toBe( + "revision-scoped", + ); + // A 401 proves the policy allowed the fixture request to reach Google. It + // does not distinguish the fixed fixture token from an unresolved + // placeholder, so this assertion intentionally makes no rewrite claim. + expect( + proof.statuses, + "Google APIs returned the expected response for the non-secret fixture request", + ).toEqual(agent === "openclaw" ? [401] : [401, 401]); + if (agent === "hermes") { + expect(proof.installedOverride, "installed Hermes Google Chat override loaded").toBe(true); + } +} + +export async function expectGooglechatProviderEgress( sandbox: SandboxClient, sandboxName: string, agent: AgentKind, @@ -112,20 +131,11 @@ export async function expectGooglechatCredentialBoundary( ? `node -e ${shellQuote(source)}` : `/opt/hermes/.venv/bin/python -c ${shellQuote(source)}`; const result = await sandboxSh(sandbox, sandboxName, command, { - artifactName: `googlechat-credential-boundary-${agent}-${context}`, + artifactName: `googlechat-provider-egress-${agent}-${context}`, redactionValues, timeoutMs: 90_000, }); - expectExitZero(result, `${agent} Google Chat credential boundary ${context}`); + expectExitZero(result, `${agent} Google Chat provider egress ${context}`); - const proof = parseBoundaryProof(result.stdout); - expect(proof.placeholder, "sandbox process received a revision-scoped placeholder").toBe( - "revision-scoped", - ); - expect(proof.statuses, "Google APIs rejected the rewritten fixed fixture credential").toEqual( - agent === "openclaw" ? [401] : [401, 401], - ); - if (agent === "hermes") { - expect(proof.installedOverride, "installed Hermes Google Chat override loaded").toBe(true); - } + assertGooglechatProviderEgressProof(parseGooglechatProviderEgressProof(result.stdout), agent); } diff --git a/test/e2e/live/channels-stop-start-helpers.ts b/test/e2e/live/channels-stop-start-helpers.ts index 1ae16ab037f..623adc92420 100644 --- a/test/e2e/live/channels-stop-start-helpers.ts +++ b/test/e2e/live/channels-stop-start-helpers.ts @@ -5,6 +5,19 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { AddSandboxChannelDependencies } from "../../../src/lib/actions/sandbox/policy-channel.ts"; +import * as policyChannelDependenciesModule from "../../../src/lib/actions/sandbox/policy-channel-dependencies.ts"; +import * as policyChannelModule from "../../../src/lib/actions/sandbox/policy-channel.ts"; +import * as openshellRuntimeModule from "../../../src/lib/adapters/openshell/runtime.ts"; +import * as messagingBridgeProviderModule from "../../../src/lib/onboard/messaging-bridge-provider.ts"; +import * as onboardProvidersModule from "../../../src/lib/onboard/providers.ts"; +import * as statePathsModule from "../../../src/lib/state/paths.ts"; +import { + assertCleanupSucceededOrAbsent, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { type OpenClawChannelConfigState, @@ -12,10 +25,13 @@ import { openClawChannelIsInert, openClawChannelStateProbeScript, } from "./channels-stop-start-config-state.ts"; +import { + channelPlanStateErrors, + type ChannelPlanExpectedState, +} from "./channels-stop-start-plan-state.ts"; import { startChannelsStopStartProgress } from "./channels-stop-start-progress.ts"; import { assertChannelsStopStartSandboxName } from "./channels-stop-start-safety.ts"; -import { GOOGLECHAT_E2E_ACCESS_TOKEN } from "./channels-stop-start-googlechat-entry.ts"; -import { expectGooglechatCredentialBoundary } from "./channels-stop-start-googlechat-proof.ts"; +import { expectGooglechatProviderEgress } from "./channels-stop-start-googlechat-proof.ts"; import { type AgentKind, runSecondaryCleanup as bestEffortPreclean, @@ -26,7 +42,6 @@ import { installSandboxOrSkipOnRateLimit, phase6Env, precleanSandbox, - REPO_ROOT, resultText, sandboxSh, shellQuote, @@ -35,6 +50,259 @@ import { } from "./phase6-messaging-helpers.ts"; import { parsePolicyPresetState } from "./policy-list-state.ts"; +type PolicyChannelModule = typeof import("../../../src/lib/actions/sandbox/policy-channel.ts"); +type PolicyChannelDependenciesModule = + typeof import("../../../src/lib/actions/sandbox/policy-channel-dependencies.ts"); +type OpenshellRuntimeModule = typeof import("../../../src/lib/adapters/openshell/runtime.ts"); +type MessagingBridgeProviderModule = + typeof import("../../../src/lib/onboard/messaging-bridge-provider.ts"); +type StatePathsModule = typeof import("../../../src/lib/state/paths.ts"); +type ProviderUpsertOptions = { + readonly replaceExisting?: boolean; + readonly revalidatePolicyRequirements?: (operation: string) => void; +}; +type ProviderDependencies = { + upsertMessagingProviders( + tokenDefs: Parameters[0], + run: typeof runOpenshell, + options?: ProviderUpsertOptions, + ): string[]; +}; + +const policyChannel = ( + "default" in policyChannelModule ? policyChannelModule.default : policyChannelModule +) as PolicyChannelModule; +const { addSandboxChannel } = policyChannel; +const policyChannelDependenciesNamespace = ( + "default" in policyChannelDependenciesModule + ? policyChannelDependenciesModule.default + : policyChannelDependenciesModule +) as PolicyChannelDependenciesModule; +const { policyChannelDependencies } = policyChannelDependenciesNamespace; +const openshellRuntime = ( + "default" in openshellRuntimeModule ? openshellRuntimeModule.default : openshellRuntimeModule +) as OpenshellRuntimeModule; +const { runOpenshell } = openshellRuntime; +const messagingBridgeProvider = ( + "default" in messagingBridgeProviderModule + ? messagingBridgeProviderModule.default + : messagingBridgeProviderModule +) as MessagingBridgeProviderModule; +const { ensureMessagingBridgeProfiles } = messagingBridgeProvider; +const onboardProviders = ( + "default" in onboardProvidersModule ? onboardProvidersModule.default : onboardProvidersModule +) as ProviderDependencies; +const statePaths = ( + "default" in statePathsModule ? statePathsModule.default : statePathsModule +) as StatePathsModule; +const { ROOT } = statePaths; + +interface GooglechatLiveE2eComposition { + readonly sandboxName: string; + readonly agent: AgentKind; + readonly audience: string; +} + +interface GooglechatLiveE2eDependencies { + readonly addSandboxChannel: ( + sandboxName: string, + options: { readonly channel: string }, + dependencies: AddSandboxChannelDependencies, + ) => Promise; + readonly installCredentialFixture: (sandboxName: string, agent: AgentKind) => () => void; + readonly rebuildSandbox?: (sandboxName: string, args: string[]) => Promise; +} + +interface GooglechatCredentialFixtureDependencies { + readonly ensureProfiles?: typeof ensureMessagingBridgeProfiles; + readonly providerDependencies?: ProviderDependencies; + readonly root?: string; + readonly run?: typeof runOpenshell; +} + +export const GOOGLECHAT_E2E_ACCESS_TOKEN = "e2e-fake-googlechat-access-token"; + +const PROVIDER_TYPE_BY_AGENT: Readonly> = { + openclaw: "google-chat-bridge", + hermes: "google-chat-hermes-bridge", +}; + +/** + * Replace Google Chat's asynchronous Google OAuth mint only inside this live-test + * helper. The fixed value is not a credential. Creating the real OpenShell + * provider with it still exercises provider identity, revision-scoped sandbox + * injection, bound provider egress, and removal without requiring a Google + * service account in CI. + */ +export function installGooglechatCredentialFixture( + sandboxName: string, + agent: AgentKind, + dependencies: GooglechatCredentialFixtureDependencies = {}, +): () => void { + assertChannelsStopStartSandboxName(sandboxName, agent); + const ensureProfiles = dependencies.ensureProfiles ?? ensureMessagingBridgeProfiles; + const providerDependencies = dependencies.providerDependencies ?? onboardProviders; + const root = dependencies.root ?? ROOT; + const run = dependencies.run ?? runOpenshell; + const expectedName = `${sandboxName}-googlechat-bridge`; + const expectedType = PROVIDER_TYPE_BY_AGENT[agent]; + const original = providerDependencies.upsertMessagingProviders; + + providerDependencies.upsertMessagingProviders = (tokenDefs, providerRun, options = {}) => { + const fixtureTokenDefs = tokenDefs.filter(({ name }) => name === expectedName); + const fixtureTokenDef = fixtureTokenDefs[0]; + if ( + fixtureTokenDefs.length !== 1 || + fixtureTokenDef?.envKey !== "GOOGLE_CHAT_ACCESS_TOKEN" || + fixtureTokenDef?.providerType !== expectedType + ) { + throw new Error("Google Chat live fixture received an unexpected provider definition"); + } + + const delegatedTokenDefs = tokenDefs.filter(({ name }) => name !== expectedName); + const delegatedProviderNames = + delegatedTokenDefs.length === 0 ? [] : original(delegatedTokenDefs, providerRun, options); + const baseRun = providerRun ?? run; + const revalidate = () => + options.revalidatePolicyRequirements?.( + `manage Google Chat live fixture provider '${expectedName}'`, + ); + const effectiveRun: typeof runOpenshell = (args, runOptions) => { + revalidate(); + return baseRun(args, runOptions); + }; + ensureProfiles(fixtureTokenDefs, { + root, + runOpenshell: effectiveRun, + redact: (value) => value.replaceAll(GOOGLECHAT_E2E_ACCESS_TOKEN, "[redacted]"), + }); + const existing = effectiveRun(["provider", "get", expectedName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + if (existing.status === 0 && options.replaceExisting) { + const removed = effectiveRun(["provider", "delete", expectedName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + if (removed.status !== 0) { + throw new Error(`Google Chat live fixture could not replace provider '${expectedName}'`); + } + } + const action = existing.status === 0 && !options.replaceExisting ? "update" : "create"; + const providerArgs = + action === "update" + ? ["provider", "update", expectedName, "--credential", "GOOGLE_CHAT_ACCESS_TOKEN"] + : [ + "provider", + "create", + "--name", + expectedName, + "--type", + expectedType, + "--credential", + "GOOGLE_CHAT_ACCESS_TOKEN", + ]; + const mutated = effectiveRun(providerArgs, { + env: { GOOGLE_CHAT_ACCESS_TOKEN: GOOGLECHAT_E2E_ACCESS_TOKEN }, + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + if (mutated.status !== 0) { + throw new Error(`Google Chat live fixture could not ${action} provider '${expectedName}'`); + } + const registered = new Set([...delegatedProviderNames, expectedName]); + return tokenDefs.map(({ name }) => name).filter((name) => registered.has(name)); + }; + + return () => { + providerDependencies.upsertMessagingProviders = original; + }; +} + +const DEFAULT_GOOGLECHAT_DEPENDENCIES: GooglechatLiveE2eDependencies = { + addSandboxChannel, + installCredentialFixture: installGooglechatCredentialFixture, + rebuildSandbox: (sandboxName, args) => + policyChannelDependencies.rebuildSandbox(sandboxName, args), +}; + +function requireLiveAudience(input: GooglechatLiveE2eComposition): string { + assertChannelsStopStartSandboxName(input.sandboxName, input.agent); + const audience = input.audience.trim(); + if (!audience) { + throw new Error("GOOGLECHAT_AUDIENCE is required for the channels-stop-start live target"); + } + return audience; +} + +async function addGooglechatWithInstalledFixture( + input: GooglechatLiveE2eComposition, + audience: string, + dependencies: GooglechatLiveE2eDependencies, +): Promise { + await dependencies.addSandboxChannel( + input.sandboxName, + { channel: "googlechat" }, + input.agent === "openclaw" + ? { + googlechatNonInteractiveAudienceCapability: Object.freeze({ audience }), + } + : {}, + ); +} + +/** Keep the fake OAuth mint installed across both provider registrations. */ +export async function addAndRebuildGooglechatForChannelsStopStartLiveE2e( + input: GooglechatLiveE2eComposition, + dependencies: GooglechatLiveE2eDependencies = DEFAULT_GOOGLECHAT_DEPENDENCIES, +): Promise { + const audience = requireLiveAudience(input); + if (!dependencies.rebuildSandbox) { + throw new Error("Google Chat live rebuild dependency is unavailable"); + } + + const restore = dependencies.installCredentialFixture(input.sandboxName, input.agent); + try { + await addGooglechatWithInstalledFixture(input, audience, dependencies); + await dependencies.rebuildSandbox(input.sandboxName, ["--yes"]); + } finally { + restore(); + } +} + +/** Keep the fake OAuth mint installed while a later lifecycle rebuild reconciles Google Chat. */ +export async function rebuildGooglechatForChannelsStopStartLiveE2e( + input: Pick, + dependencies: GooglechatLiveE2eDependencies = DEFAULT_GOOGLECHAT_DEPENDENCIES, +): Promise { + if (!dependencies.rebuildSandbox) { + throw new Error("Google Chat live rebuild dependency is unavailable"); + } + + const restore = dependencies.installCredentialFixture(input.sandboxName, input.agent); + try { + await dependencies.rebuildSandbox(input.sandboxName, ["--yes"]); + } finally { + restore(); + } +} + +async function withLiveE2eEnvironment( + env: NodeJS.ProcessEnv, + operation: () => Promise, +): Promise { + const original = { ...process.env }; + for (const key of Object.keys(process.env)) delete process.env[key]; + Object.assign(process.env, env); + try { + return await operation(); + } finally { + for (const key of Object.keys(process.env)) delete process.env[key]; + Object.assign(process.env, original); + } +} + const AGENT = (process.env.NEMOCLAW_CHANNELS_STOP_START_AGENT ?? process.env.NEMOCLAW_AGENT ?? "openclaw") as AgentKind; @@ -64,6 +332,58 @@ const PROVIDERS: Record string[]> = { teams: (sandbox) => [`${sandbox}-teams-bridge`], googlechat: (sandbox) => [`${sandbox}-googlechat-bridge`], }; +const PROVIDER_ALREADY_ABSENT = + /\bNotFound\b|provider[^\n]*(?:not found|does not exist)|no (?:such )?provider/i; + +function channelsStopStartProviderNames(sandboxName: string): string[] { + return CHANNELS.flatMap((channel) => PROVIDERS[channel](sandboxName)); +} + +async function cleanupChannelsStopStartProvider( + host: HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + provider: string, +): Promise { + const result = await host.command(host.openshellCommandPath, ["provider", "delete", provider], { + artifactName: `cleanup-channels-stop-start-openshell-provider-delete-${provider}`, + env, + redactionValues: redactions, + timeoutMs: 60_000, + }); + assertCleanupSucceededOrAbsent( + result, + PROVIDER_ALREADY_ABSENT, + `cleanup OpenShell provider ${provider}`, + ); +} + +export function registerChannelsStopStartProviderCleanup( + cleanup: CleanupRegistry, + host: HostCliClient, + options: { + readonly agent: AgentKind; + readonly env: NodeJS.ProcessEnv; + readonly redactions: string[]; + readonly sandboxName: string; + }, +): void { + assertChannelsStopStartSandboxName(options.sandboxName, options.agent); + for (const provider of channelsStopStartProviderNames(options.sandboxName)) { + cleanup.trackDisposable(`delete OpenShell provider ${provider}`, () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: `cleanup-channels-stop-start-probe-openshell-provider-${provider}`, + env: options.env, + redactionValues: options.redactions, + timeoutMs: 30_000, + }, + () => cleanupChannelsStopStartProvider(host, options.env, options.redactions, provider), + ), + ); + } +} // Channels that emit no credentialBinding, each for its own reason. Independent oracle — // hardcoded on purpose, not derived from the manifest under test (that would be circular). const CHANNELS_WITHOUT_CREDENTIAL_BINDING: Record = { @@ -72,7 +392,6 @@ const CHANNELS_WITHOUT_CREDENTIAL_BINDING: Record = { }; export const LIVE_TIMEOUT_MS = 80 * 60_000; -type ChannelState = "active" | "disabled"; type AgentConfigState = "active" | "inert"; type JsonRecord = Record; type Phase6Tokens = { @@ -187,12 +506,6 @@ function arrayRecords(value: unknown): JsonRecord[] { : []; } -function stringArray(value: unknown): string[] { - return Array.isArray(value) - ? value.filter((item): item is string => typeof item === "string") - : []; -} - function readRegistryEntry(sandboxName: string): JsonRecord { expect(fs.existsSync(REGISTRY_FILE), `${REGISTRY_FILE} missing`).toBe(true); const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { @@ -229,75 +542,17 @@ function planChannel(channelId: string) { ); } -function expectPlanChannelState(channelId: string, expected: ChannelState): void { - const plan = messagingPlan(SANDBOX_NAME); - const channels = arrayRecords(plan.channels); - const channel = channels.find((entry) => entry.channelId === channelId); - expect(channel, `${channelId} missing from messaging.plan.channels`).toBeTruthy(); - expect(channel?.configured, `${channelId} configured`).toBe(true); - expect(plan.sandboxName, "messaging.plan.sandboxName").toBe(SANDBOX_NAME); - expect(plan.agent, "messaging.plan.agent").toBe(AGENT); - - const disabledChannels = stringArray(plan.disabledChannels); - if (expected === "active") { - expect(channel?.active, `${channelId} active`).toBe(true); - expect(channel?.disabled, `${channelId} disabled unexpectedly`).not.toBe(true); - expect(disabledChannels, `${channelId} unexpectedly disabled`).not.toContain(channelId); - } else { - expect(channel?.disabled, `${channelId} disabled`).toBe(true); - expect(channel?.active, `${channelId} active unexpectedly`).not.toBe(true); - expect(disabledChannels, `${channelId} missing from disabledChannels`).toContain(channelId); - } - - const networkPolicy = - plan.networkPolicy && typeof plan.networkPolicy === "object" - ? (plan.networkPolicy as Record) - : {}; - expect(stringArray(networkPolicy.presets), `${channelId} policy preset`).toContain(channelId); - expect( - arrayRecords(networkPolicy.entries).some((entry) => entry.channelId === channelId), - `${channelId} policy entry`, - ).toBe(true); - const credentialBindings = arrayRecords(plan.credentialBindings); - if (!Object.hasOwn(CHANNELS_WITHOUT_CREDENTIAL_BINDING, channelId)) { - expect( - credentialBindings.some((entry) => entry.channelId === channelId), - `${channelId} credential binding`, - ).toBe(true); - } - expect(Object.hasOwn(plan, "agentRender"), "messaging.plan.agentRender should not persist").toBe( - false, - ); - expect( - channels.some((entry) => Object.hasOwn(entry, "hooks")), - "messaging.plan.channels hooks should not persist", - ).toBe(false); -} - -function expectPlanChannelRemoved(channelId: string): void { - const plan = messagingPlan(SANDBOX_NAME); - expect( - arrayRecords(plan.channels).some((channel) => channel.channelId === channelId), - `${channelId} remained in messaging.plan.channels`, - ).toBe(false); - expect(stringArray(plan.disabledChannels), `${channelId} remained disabled`).not.toContain( - channelId, - ); - const networkPolicy = - plan.networkPolicy && typeof plan.networkPolicy === "object" - ? (plan.networkPolicy as Record) - : {}; - expect(stringArray(networkPolicy.presets), `${channelId} policy preset remained`).not.toContain( - channelId, - ); +function expectPlanChannelState(channelId: string, expected: ChannelPlanExpectedState): void { expect( - arrayRecords(networkPolicy.entries).some((entry) => entry.channelId === channelId), - `${channelId} policy entry remained`, - ).toBe(false); - expect( - arrayRecords(plan.credentialBindings).some((entry) => entry.channelId === channelId), - `${channelId} credential binding remained`, - ).toBe(false); + channelPlanStateErrors(messagingPlan(SANDBOX_NAME), { + agent: AGENT, + channelId, + credentialBindingRequired: !Object.hasOwn(CHANNELS_WITHOUT_CREDENTIAL_BINDING, channelId), + expected, + sandboxName: SANDBOX_NAME, + }), + `${channelId} ${expected} persisted messaging plan contract`, + ).toEqual([]); } function requireEnvValue(env: NodeJS.ProcessEnv, key: string): string { @@ -550,17 +805,12 @@ async function addGooglechatForLiveE2e( env: NodeJS.ProcessEnv, redactions: string[], ): Promise { - const entrypoint = path.join(REPO_ROOT, "test/e2e/live/channels-stop-start-googlechat-entry.ts"); - const tsx = path.join(REPO_ROOT, "node_modules/tsx/dist/cli.mjs"); - const addAndRebuild = await host.command("node", [tsx, entrypoint, SANDBOX_NAME], { - artifactName: "channels-stop-start-add-and-rebuild-googlechat-live-e2e", - env, - redactionValues: redactions, - timeoutMs: 10 * 60_000, - }); - expectExitZero( - addAndRebuild, - "add and rebuild Google Chat through live-E2E capability composition", + await withLiveE2eEnvironment(env, () => + addAndRebuildGooglechatForChannelsStopStartLiveE2e({ + sandboxName: SANDBOX_NAME, + agent: AGENT, + audience: env.GOOGLECHAT_AUDIENCE ?? "", + }), ); await expectSandboxReady( host, @@ -571,19 +821,13 @@ async function addGooglechatForLiveE2e( ); } -async function rebuildWithGooglechatFixtureForLiveE2e( - host: import("../fixtures/clients/host.ts").HostCliClient, - env: NodeJS.ProcessEnv, - redactions: string[], -) { - const entrypoint = path.join(REPO_ROOT, "test/e2e/live/channels-stop-start-googlechat-entry.ts"); - const tsx = path.join(REPO_ROOT, "node_modules/tsx/dist/cli.mjs"); - return host.command("node", [tsx, entrypoint, SANDBOX_NAME, "--rebuild-only"], { - artifactName: `rebuild-start-all-${AGENT}`, - env, - redactionValues: redactions, - timeoutMs: 30 * 60_000, - }); +async function rebuildWithGooglechatFixtureForLiveE2e(env: NodeJS.ProcessEnv): Promise { + await withLiveE2eEnvironment(env, () => + rebuildGooglechatForChannelsStopStartLiveE2e({ + sandboxName: SANDBOX_NAME, + agent: AGENT, + }), + ); } async function policyPresetState( @@ -655,7 +899,7 @@ async function removeGooglechatAndRebuild( ); expectExitZero(remove, "channels remove googlechat"); expect(resultText(remove)).toContain("Removed googlechat"); - expectPlanChannelRemoved("googlechat"); + expectPlanChannelState("googlechat", "removed"); const rebuild = await rebuildSandbox( host, @@ -700,7 +944,7 @@ export async function runChannelsStopStartTarget({ await artifacts.target.declare({ id: "channels-stop-start", boundary: - "messaging onboard + channel lifecycle + Google Chat provider cleanup + revision-scoped outbound credential rewrite + installed Hermes pull/ack", + "messaging onboard + channel lifecycle + Google Chat provider cleanup + revision-scoped placeholder and provider egress + installed Hermes pull/ack", agent: AGENT, sandboxName: SANDBOX_NAME, channels: CHANNELS, @@ -724,6 +968,12 @@ export async function runChannelsStopStartTarget({ redactions, `cleanup-channels-stop-start-${AGENT}`, ); + registerChannelsStopStartProviderCleanup(cleanup, host, { + agent: AGENT, + env, + redactions, + sandboxName: SANDBOX_NAME, + }); await precleanSandbox( host, SANDBOX_NAME, @@ -765,7 +1015,7 @@ export async function runChannelsStopStartTarget({ expectChannelInputs(env); for (const channel of CHANNELS) expectPlanChannelState(channel, "active"); await expectAgentConfig(sandbox, "active", "baseline", redactions); - await expectGooglechatCredentialBoundary(sandbox, SANDBOX_NAME, AGENT, "baseline", redactions); + await expectGooglechatProviderEgress(sandbox, SANDBOX_NAME, AGENT, "baseline", redactions); await expectProvidersExist(host, env, redactions, "baseline"); for (const channel of CHANNELS) { expect( @@ -801,11 +1051,10 @@ export async function runChannelsStopStartTarget({ for (const channel of CHANNELS) await runChannelCommand(host, env, redactions, "start", channel); expectChannelInputs(env); for (const channel of CHANNELS) expectPlanChannelState(channel, "active"); - const startRebuild = await rebuildWithGooglechatFixtureForLiveE2e(host, env, redactions); - expectExitZero(startRebuild, "rebuild after starting all channels"); + await rebuildWithGooglechatFixtureForLiveE2e(env); expectChannelInputs(env); await expectAgentConfig(sandbox, "active", "after-start", redactions); - await expectGooglechatCredentialBoundary(sandbox, SANDBOX_NAME, AGENT, "after-start", redactions); + await expectGooglechatProviderEgress(sandbox, SANDBOX_NAME, AGENT, "after-start", redactions); await expectProvidersExist(host, env, redactions, "after-start"); for (const channel of CHANNELS) expectPlanChannelState(channel, "active"); for (const channel of CHANNELS) { @@ -817,7 +1066,7 @@ export async function runChannelsStopStartTarget({ progress.phase("remove Google Chat and validate provider cleanup"); await removeGooglechatAndRebuild(host, env, redactions); - expectPlanChannelRemoved("googlechat"); + expectPlanChannelState("googlechat", "removed"); await expectChannelProvidersAbsent(host, env, redactions, "googlechat", "after-remove"); expect( await policyPresetState(host, env, redactions, "googlechat", "after-remove"), diff --git a/test/e2e/live/channels-stop-start-plan-state.ts b/test/e2e/live/channels-stop-start-plan-state.ts new file mode 100644 index 00000000000..39abdf5c31b --- /dev/null +++ b/test/e2e/live/channels-stop-start-plan-state.ts @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { parseSandboxMessagingPlan } from "../../../src/lib/messaging/plan-validation.ts"; +import type { AgentKind } from "./phase6-messaging-helpers.ts"; + +export type ChannelPlanExpectedState = "active" | "disabled" | "removed"; + +export type ChannelPlanStateExpectation = { + readonly agent: AgentKind; + readonly channelId: string; + readonly credentialBindingRequired: boolean; + readonly expected: ChannelPlanExpectedState; + readonly sandboxName: string; +}; + +export function channelPlanStateErrors( + value: unknown, + expectation: ChannelPlanStateExpectation, +): string[] { + const plan = parseSandboxMessagingPlan(value, { + agent: expectation.agent, + sandboxName: expectation.sandboxName, + }); + if (!plan) { + return [ + `messaging.plan must be a valid persisted plan for ${expectation.sandboxName} using ${expectation.agent}`, + ]; + } + + const errors: string[] = []; + const persistedPlan = value as Record; + if (Object.hasOwn(persistedPlan, "agentRender")) { + errors.push("messaging.plan.agentRender must not persist"); + } + + const persistedChannels = persistedPlan.channels as Record[]; + if (persistedChannels.some((channel) => Object.hasOwn(channel, "hooks"))) { + errors.push("messaging.plan channel hooks must not persist"); + } + const channel = plan.channels.find((entry) => entry.channelId === expectation.channelId); + const disabledChannels = plan.disabledChannels; + const policyPresets = plan.networkPolicy.presets; + const policyEntries = plan.networkPolicy.entries; + const credentialBindings = (persistedPlan.credentialBindings ?? []) as { + channelId: string; + }[]; + const hasPolicyEntry = policyEntries.some((entry) => entry.channelId === expectation.channelId); + const hasCredentialBinding = credentialBindings.some( + (entry) => entry.channelId === expectation.channelId, + ); + + if (expectation.expected === "removed") { + if (channel) + errors.push(`${expectation.channelId} must be absent from messaging.plan.channels`); + if (disabledChannels.includes(expectation.channelId)) { + errors.push(`${expectation.channelId} must be absent from disabledChannels`); + } + if (policyPresets.includes(expectation.channelId)) { + errors.push(`${expectation.channelId} policy preset must be removed`); + } + if (hasPolicyEntry) errors.push(`${expectation.channelId} policy entry must be removed`); + if (hasCredentialBinding) { + errors.push(`${expectation.channelId} credential binding must be removed`); + } + return errors; + } + + if (!channel) { + errors.push(`${expectation.channelId} must be present in messaging.plan.channels`); + } else { + if (channel.configured !== true) errors.push(`${expectation.channelId} must be configured`); + if (expectation.expected === "active") { + if (channel.active !== true) errors.push(`${expectation.channelId} must be active`); + if (channel.disabled === true) errors.push(`${expectation.channelId} must not be disabled`); + } else { + if (channel.disabled !== true) errors.push(`${expectation.channelId} must be disabled`); + if (channel.active === true) errors.push(`${expectation.channelId} must not be active`); + } + } + + if (expectation.expected === "active" && disabledChannels.includes(expectation.channelId)) { + errors.push(`${expectation.channelId} must be absent from disabledChannels while active`); + } + if (expectation.expected === "disabled" && !disabledChannels.includes(expectation.channelId)) { + errors.push(`${expectation.channelId} must be present in disabledChannels while disabled`); + } + if (!policyPresets.includes(expectation.channelId)) { + errors.push(`${expectation.channelId} policy preset must be present`); + } + if (!hasPolicyEntry) errors.push(`${expectation.channelId} policy entry must be present`); + if (expectation.credentialBindingRequired && !hasCredentialBinding) { + errors.push(`${expectation.channelId} credential binding must be present`); + } + return errors; +} diff --git a/test/e2e/live/hermes-slack-credential-transport.ts b/test/e2e/live/hermes-slack-credential-transport.ts new file mode 100644 index 00000000000..9fa40cc2664 --- /dev/null +++ b/test/e2e/live/hermes-slack-credential-transport.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { shellQuote } from "../fixtures/clients/command.ts"; + +export interface HermesSlackCredentialFingerprint { + readonly byteLength: number; + readonly sha256: string; +} + +export function hermesSlackCredentialFingerprints( + credentials: readonly string[], +): readonly HermesSlackCredentialFingerprint[] { + const fingerprints = credentials + .filter((credential) => credential.length > 0) + .map((credential) => ({ + byteLength: Buffer.byteLength(credential), + sha256: createHash("sha256").update(credential).digest("hex"), + })); + if (fingerprints.length === 0) { + throw new Error("Hermes Slack credential scan requires at least one credential fingerprint"); + } + return fingerprints; +} + +export const HERMES_SLACK_CREDENTIAL_FINGERPRINT_SCAN_SOURCE = String.raw` +import hashlib +import json +import pathlib +import sys + +fingerprints = json.load(sys.stdin) +file_paths = json.loads(sys.argv[1]) +scan_processes = sys.argv[2] == "processes" + +def contains_fingerprint(data): + for fingerprint in fingerprints: + width = fingerprint.get("byteLength") + expected = fingerprint.get("sha256") + if not isinstance(width, int) or width <= 0: + raise RuntimeError("credential fingerprint byteLength is invalid") + if not isinstance(expected, str) or len(expected) != 64: + raise RuntimeError("credential fingerprint sha256 is invalid") + for offset in range(max(0, len(data) - width + 1)): + if hashlib.sha256(data[offset:offset + width]).hexdigest() == expected: + return True + return False + +file_hit = False +for raw_path in file_paths: + try: + data = pathlib.Path(raw_path).read_bytes() + except Exception: + continue + if contains_fingerprint(data): + file_hit = True + break + +process_hit = False +process_observed = False +if scan_processes: + for path in pathlib.Path("/proc").glob("[0-9]*/cmdline"): + try: + data = path.read_bytes() + except Exception: + continue + if not data: + continue + process_observed = True + if contains_fingerprint(data): + process_hit = True + break + +print(json.dumps({ + "files": "LEAK" if file_hit else "OK", + "processes": "LEAK" if process_hit else ("OK" if process_observed else "EMPTY"), +}, sort_keys=True)) +`; + +export function hermesSlackCredentialFingerprintScanCommand(filePaths: readonly string[]): string { + return [ + "python3 -c", + shellQuote(HERMES_SLACK_CREDENTIAL_FINGERPRINT_SCAN_SOURCE), + shellQuote(JSON.stringify(filePaths)), + "processes", + ].join(" "); +} + +export function hermesSlackCredentialScanScript(options: { + credentialFingerprints: readonly HermesSlackCredentialFingerprint[]; + openshellCommandPath: string; + remoteCommand: string; + sandboxName: string; +}): string { + if (options.credentialFingerprints.length === 0) { + throw new Error("Hermes Slack credential scan requires at least one credential fingerprint"); + } + const fingerprintPayload = JSON.stringify(options.credentialFingerprints); + return [ + "set -euo pipefail", + [ + "printf %s", + shellQuote(fingerprintPayload), + "|", + shellQuote(options.openshellCommandPath), + "sandbox exec --name", + shellQuote(options.sandboxName), + "-- sh -lc", + shellQuote(options.remoteCommand), + ].join(" "), + ].join("\n"); +} diff --git a/test/e2e/live/hermes-slack-e2e-helpers.ts b/test/e2e/live/hermes-slack-e2e-helpers.ts index 035e5b3c0c6..7d6061b37cc 100644 --- a/test/e2e/live/hermes-slack-e2e-helpers.ts +++ b/test/e2e/live/hermes-slack-e2e-helpers.ts @@ -2,11 +2,21 @@ // SPDX-License-Identifier: Apache-2.0 import { setTimeout as sleep } from "node:timers/promises"; -import { cleanupWhenOpenShellAvailable } from "../fixtures/cleanup-resources.ts"; +import { + assertCleanupSucceededOrAbsent, + cleanupWhenOpenShellAvailable, + registerSandboxCleanupUnlessKept, +} from "../fixtures/cleanup-resources.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { type E2ETargetFixtures, expect } from "../fixtures/e2e-test.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + hermesSlackCredentialFingerprintScanCommand, + hermesSlackCredentialFingerprints, + hermesSlackCredentialScanScript, +} from "./hermes-slack-credential-transport.ts"; +import { assertHermesSlackApiProof } from "./hermes-slack-proof.ts"; import { runSecondaryCleanup as bestEffortLifecycleCleanup, CLI, @@ -22,9 +32,24 @@ import { trackPreinstallSandboxCleanup, } from "./phase6-messaging-helpers.ts"; -const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-slack"; +const HERMES_SLACK_E2E_SANDBOX_NAME = "e2e-hermes-slack"; + +export function assertHermesSlackSandboxName(sandboxName: string): void { + if (sandboxName !== HERMES_SLACK_E2E_SANDBOX_NAME) { + throw new Error( + `Hermes Slack live test is destructive and only accepts sandbox name ${HERMES_SLACK_E2E_SANDBOX_NAME}; got ${sandboxName}`, + ); + } +} + +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? HERMES_SLACK_E2E_SANDBOX_NAME; +assertHermesSlackSandboxName(SANDBOX_NAME); const SLACK_BOT_TOKEN = process.env.SLACK_BOT_TOKEN ?? "xoxb-test-hermes-slack-token"; const SLACK_APP_TOKEN = process.env.SLACK_APP_TOKEN ?? "xapp-test-hermes-slack-app-token"; +const SLACK_CREDENTIAL_FINGERPRINTS = hermesSlackCredentialFingerprints([ + SLACK_BOT_TOKEN, + SLACK_APP_TOKEN, +]); export const LIVE_TIMEOUT_MS = 70 * 60_000; const INSTALL_TIMEOUT_MS = 60 * 60_000; const HERMES_HEALTH_URL = "http://localhost:8642/health"; @@ -125,46 +150,45 @@ async function cleanupHermesSlackProvider(options: { timeoutMs: 60_000, }, ); - if ( - result.exitCode === 0 || - /\bNotFound\b|provider[^\n]*(?:not found|does not exist)|no such provider/i.test( - resultText(result), - ) - ) { - return; - } - expectExitZero(result, `cleanup OpenShell provider ${options.provider}`); + assertCleanupSucceededOrAbsent( + result, + /\bNotFound\b|provider[^\n]*(?:not found|does not exist)|no (?:such )?provider/i, + `cleanup OpenShell provider ${options.provider}`, + ); } -async function hostSlackTokenStdin(options: { +export function assertHermesSlackCredentialFingerprintScanResult(result: { + readonly files?: string; + readonly processes?: string; +}): void { + expect(result.files, "raw Slack token absent from selected files and logs").toBe("OK"); + expect(result.processes, "raw Slack token absent from process arguments").toBe("OK"); +} + +async function scanHermesSlackCredentialFingerprints(options: { host: HostCliClient; apiKey: string; artifactName: string; - remoteCommand: string; timeoutMs?: number; }): Promise { - const script = [ - "set -euo pipefail", - "ssh_config=$(mktemp)", - "trap 'rm -f \"$ssh_config\"' EXIT", - `${shellQuote(options.host.openshellCommandPath)} sandbox ssh-config ${shellQuote(SANDBOX_NAME)} >"$ssh_config"`, - [ - 'printf "%s\\n%s\\n" "$SLACK_BOT_TOKEN" "$SLACK_APP_TOKEN"', - "|", - "ssh", - '-F "$ssh_config"', - "-o StrictHostKeyChecking=no", - "-o UserKnownHostsFile=/dev/null", - "-o ConnectTimeout=10", - "-o LogLevel=ERROR", - shellQuote(`openshell-${SANDBOX_NAME}.default`), - shellQuote(options.remoteCommand), - ].join(" "), - ].join("\n"); + const scanEnv = hermesSlackEnv(options.apiKey); + delete scanEnv.SLACK_APP_TOKEN; + delete scanEnv.SLACK_BOT_TOKEN; + const script = hermesSlackCredentialScanScript({ + credentialFingerprints: SLACK_CREDENTIAL_FINGERPRINTS, + openshellCommandPath: options.host.openshellCommandPath, + remoteCommand: hermesSlackCredentialFingerprintScanCommand([ + "/sandbox/.hermes/config.yaml", + "/sandbox/.hermes/.env", + "/tmp/nemoclaw-start.log", + "/tmp/gateway.log", + ]), + sandboxName: SANDBOX_NAME, + }); return options.host.command("bash", ["-lc", script], { artifactName: options.artifactName, - env: hermesSlackEnv(options.apiKey), + env: scanEnv, redactionValues: redactions(options.apiKey), timeoutMs: options.timeoutMs ?? 60_000, }); @@ -237,6 +261,70 @@ type HermesSlackE2EFixtures = E2ETargetFixtures & { skip: (note?: string) => never; }; +export function registerHermesSlackCleanup( + { cleanup, host, sandbox }: Pick, + options: { + apiKey: string; + env: NodeJS.ProcessEnv; + keepSandbox: boolean; + redactionValues: string[]; + sandboxName: string; + }, +): void { + assertHermesSlackSandboxName(options.sandboxName); + registerSandboxCleanupUnlessKept(options.keepSandbox, () => { + const gatewayCleanupOptions = { + artifactName: "cleanup-hermes-slack-openshell-gateway-destroy", + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: 120_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-hermes-slack-probe-openshell-gateway", + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + for (const provider of [ + `${options.sandboxName}-slack-app`, + `${options.sandboxName}-slack-bridge`, + ]) { + cleanup.trackDisposable(`delete OpenShell provider ${provider}`, () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: `cleanup-hermes-slack-probe-openshell-provider-${provider}`, + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: 30_000, + }, + () => cleanupHermesSlackProvider({ host, apiKey: options.apiKey, provider }), + ), + ); + } + trackPreinstallSandboxCleanup( + cleanup, + host, + sandbox, + options.sandboxName, + options.env, + options.redactionValues, + "cleanup-hermes-slack", + ); + }); +} + export async function runHermesSlackE2E({ artifacts, cleanup, @@ -250,51 +338,15 @@ export async function runHermesSlackE2E({ const env = hermesSlackEnv(apiKey); const redactionValues = redactions(apiKey); - const gatewayCleanupOptions = { - artifactName: "cleanup-hermes-slack-openshell-gateway-destroy", - env, - redactionValues, - timeoutMs: 120_000, - }; - cleanup.trackGateway( + registerHermesSlackCleanup( + { cleanup, host, sandbox }, { - cleanupGatewayRegistration: (name: string) => - cleanupWhenOpenShellAvailable( - host, - { - artifactName: "cleanup-hermes-slack-probe-openshell-gateway", - env, - redactionValues, - timeoutMs: 30_000, - }, - () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), - ), + apiKey, + env, + keepSandbox: process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1", + redactionValues, + sandboxName: SANDBOX_NAME, }, - "nemoclaw", - gatewayCleanupOptions, - ); - for (const provider of [`${SANDBOX_NAME}-slack-app`, `${SANDBOX_NAME}-slack-bridge`]) { - cleanup.trackDisposable(`delete OpenShell provider ${provider}`, () => - cleanupWhenOpenShellAvailable( - host, - { - artifactName: `cleanup-hermes-slack-probe-openshell-provider-${provider}`, - env, - redactionValues, - timeoutMs: 30_000, - }, - () => cleanupHermesSlackProvider({ host, apiKey, provider }), - ), - ); - } - trackPreinstallSandboxCleanup( - cleanup, - host, - sandbox, - SANDBOX_NAME, - env, - redactionValues, - "cleanup-hermes-slack", ); await artifacts.target.declare({ @@ -556,38 +608,17 @@ PY`, expectExitZero(secretBoundaryProbe, "Hermes Slack secret-boundary scan"); expect(secretBoundaryProbe.stdout.trim()).toBe("OK"); - const tokenFileHits = await hostSlackTokenStdin({ + const tokenScan = await scanHermesSlackCredentialFingerprints({ host, apiKey, - artifactName: "phase-4-token-file-hits", - remoteCommand: - "grep -Fq -f - /sandbox/.hermes/config.yaml /sandbox/.hermes/.env /tmp/nemoclaw-start.log /tmp/gateway.log 2>/dev/null && echo LEAK || echo OK", + artifactName: "phase-4-token-fingerprint-scan", }); - expectExitZero(tokenFileHits, "raw Slack token file/log scan"); - expect(tokenFileHits.stdout.trim()).toBe("OK"); - - const processScan = await hostSlackTokenStdin({ - host, - apiKey, - artifactName: "phase-4-token-process-scan", - remoteCommand: String.raw`python3 -c 'import pathlib, sys -raw_tokens = {line.rstrip("\n") for line in sys.stdin if line.rstrip("\n")} -cmdlines = [] -for path in pathlib.Path("/proc").glob("[0-9]*/cmdline"): - try: - cmdlines.append(path.read_bytes().replace(b"\0", b"\n").decode("utf-8", "replace")) - except Exception: - pass -text = "\n".join(cmdlines) -if not text: - print("EMPTY") -elif any(token in text for token in raw_tokens): - print("LEAK") -else: - print("OK")'`, - }); - expectExitZero(processScan, "raw Slack token process scan"); - if (processScan.stdout.trim() !== "EMPTY") expect(processScan.stdout.trim()).toBe("OK"); + expectExitZero(tokenScan, "raw Slack credential fingerprint scan"); + const tokenScanResult = JSON.parse(tokenScan.stdout.trim()) as { + files?: string; + processes?: string; + }; + assertHermesSlackCredentialFingerprintScanResult(tokenScanResult); progress.phase("validate Hermes-scoped Slack policy"); const policy = await host.command( @@ -640,7 +671,7 @@ done`, expectExitZero(bridgeResidue, "Hermes Slack bridge residue probe"); expect(resultText(bridgeResidue).trim()).toBe(""); - progress.phase("exercise Slack API through credential aliases"); + progress.phase("exercise Slack API egress with credential placeholders"); const slackProbe = await sandboxShWithArgs( sandbox, SANDBOX_NAME, @@ -656,8 +687,9 @@ import sys import urllib.error import urllib.request -# Verified: this probe crosses the credential-rewrite boundary, so the proxy -# chain has to be trusted, not just reachable. +# This probe sends the revision-scoped placeholders through permitted Slack +# provider egress. Upstream authentication responses prove reachability; the +# messaging-providers target owns capture-based credential-rewrite proof. TLS_CONTEXT = ssl.create_default_context() INJECTED_RE = re.compile(r"^openshell:resolve:env:(v[0-9]+_)?(SLACK_BOT_TOKEN|SLACK_APP_TOKEN)$") @@ -750,14 +782,8 @@ PY`, ); const slackProbeText = resultText(slackProbe); await artifacts.writeText("phase-6-slack-python-probe.txt", slackProbeText); - if (/^TIMEOUT/m.test(slackProbeText)) { - skip("Slack API timed out"); - return; - } + assertHermesSlackApiProof(slackProbeText); expectExitZero(slackProbe, "Slack Python API probe"); - expect(slackProbeText).toMatch(/^OK auth\.test:/m); - expect(slackProbeText).toMatch(/^OK apps\.connections\.open:/m); - expect(slackProbeText).not.toMatch(/^(FAIL|ERROR)/m); if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { progress.phase("remove Hermes Slack sandbox"); @@ -818,7 +844,7 @@ PY`, rawTokensAbsentFromFilesLogsAndProcesses: true, hermesScopedSlackPolicy: true, noLegacyDecodeBridgeResidue: true, - slackPythonAliasEgress: true, + slackPythonProviderEgress: true, cleanupVerified: process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1", }, }); diff --git a/test/e2e/live/hermes-slack-e2e.test.ts b/test/e2e/live/hermes-slack-e2e.test.ts index 2c06740be40..c30c122fc3e 100644 --- a/test/e2e/live/hermes-slack-e2e.test.ts +++ b/test/e2e/live/hermes-slack-e2e.test.ts @@ -16,7 +16,7 @@ test( "validate Slack providers and Hermes health", "inspect Slack config and secret isolation", "validate Hermes-scoped Slack policy", - "exercise Slack API through credential aliases", + "exercise Slack API egress with credential placeholders", "remove Hermes Slack sandbox", "record Hermes Slack results", ], diff --git a/test/e2e/live/hermes-slack-proof.ts b/test/e2e/live/hermes-slack-proof.ts new file mode 100644 index 00000000000..d1ce0fa8b05 --- /dev/null +++ b/test/e2e/live/hermes-slack-proof.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type HermesSlackApiProof = + | { readonly kind: "passed" } + | { readonly kind: "timeout"; readonly reason: string } + | { readonly kind: "failed"; readonly reason: string }; + +export function classifyHermesSlackApiProof(output: string): HermesSlackApiProof { + const timeout = output.match(/^TIMEOUT[^\r\n]*/mu)?.[0]; + if (timeout) return { kind: "timeout", reason: timeout }; + + const failure = output.match(/^(?:FAIL|ERROR)[^\r\n]*/mu)?.[0]; + if (failure) return { kind: "failed", reason: failure }; + + const expectedMarkers: ReadonlyArray = [ + ["auth.test", /^OK auth[.]test:/mu], + ["apps.connections.open", /^OK apps[.]connections[.]open:/mu], + ]; + const missing = expectedMarkers.flatMap(([label, pattern]) => + pattern.test(output) ? [] : [label], + ); + if (missing.length > 0) { + return { kind: "failed", reason: `missing successful probe marker: ${missing.join(", ")}` }; + } + return { kind: "passed" }; +} + +export function assertHermesSlackApiProof(output: string): void { + const proof = classifyHermesSlackApiProof(output); + if (proof.kind === "passed") return; + + const result = proof.kind === "timeout" ? "incomplete required evidence" : "failed"; + throw new Error(`Slack API proof ${result}: ${proof.reason}`); +} diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 78fc753cc95..d5728e2f768 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -113,6 +113,9 @@ "test/e2e/support/hermes-gpu-startup-integrity.test.ts", "test/e2e/support/hermes-gpu-startup-proof.test.ts", "test/e2e/support/hermes-workflow-boundary.test.ts", + "test/e2e/support/managed-image-cohort-contract.test.ts", + "test/e2e/support/managed-image-receipt.test.ts", + "test/e2e/support/stock-managed-image-workflow-boundary.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -164,6 +167,7 @@ "test/e2e/support/e2e-semantic-phase-check.test.ts", "test/e2e/support/workflow-e2e-progress.test.ts", "test/e2e/support/prepare-e2e-workflow-boundary.test.ts", + "test/e2e/support/openclaw-agent-assertion.test.ts", "test/install/installer-hash-check.test.ts", "test/e2e-runtime/runner.test.ts" ] @@ -187,11 +191,15 @@ "fast": [ "src/lib/actions/sandbox/supervisor-relaunch.test.ts", "src/lib/onboard/docker-gpu-patch-finalize.test.ts", + "src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts", "src/lib/onboard/docker-startup-command-patch.test.ts", "src/lib/sandbox/privileged-exec.test.ts", "test/inference/managed/managed-gateway-control.test.ts", "test/agents/openclaw/runtime/nemoclaw-start-guard-recovery.test.ts", "test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts", + "test/e2e/support/managed-image-cohort-contract.test.ts", + "test/e2e/support/managed-image-receipt.test.ts", + "test/e2e/support/stock-managed-image-workflow-boundary.test.ts", "test/process-recovery/process-recovery-supervisor-relaunch.test.ts", "test/onboarding/docker-final-handoff-lifecycle.test.ts" ] @@ -199,6 +207,8 @@ { "live": "test/e2e/live/network-policy.test.ts", "fast": [ + "test/channels/channels-add-preset.test.ts", + "test/runtime/policy/policy-channel-agent-resolution.test.ts", "test/onboarding/validate-blueprint.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" @@ -318,6 +328,9 @@ { "live": "test/e2e/live/channels-add-remove.test.ts", "fast": [ + "test/channels/channels-add-bridge-lifecycle.test.ts", + "test/channels/channels-add-preset.test.ts", + "test/channels/channels-remove-full-teardown.test.ts", "test/e2e/support/channels-add-remove-helpers.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" @@ -335,6 +348,9 @@ "fast": [ "src/lib/onboard/initial-policy-real-policy.test.ts", "src/lib/onboard/managed-startup-image-runtime-handoff.test.ts", + "test/e2e/support/managed-image-cohort-contract.test.ts", + "test/e2e/support/managed-image-receipt.test.ts", + "test/e2e/support/stock-managed-image-workflow-boundary.test.ts", "test/e2e/support/corporate-ca-workload-kind.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" @@ -372,6 +388,10 @@ { "live": "test/e2e/live/double-onboard.test.ts", "fast": [ + "src/lib/onboard/created-sandbox-finalization.test.ts", + "src/lib/onboard/machine/core-flow-phases.test.ts", + "src/lib/onboard/machine/handlers/sandbox-route-publication.test.ts", + "src/lib/onboard/sandbox-registration.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -386,6 +406,9 @@ { "live": "test/e2e/live/gpu-e2e.test.ts", "fast": [ + "test/e2e/support/managed-image-cohort-contract.test.ts", + "test/e2e/support/managed-image-receipt.test.ts", + "test/e2e/support/stock-managed-image-workflow-boundary.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -393,6 +416,11 @@ { "live": "test/e2e/live/hermes-discord.test.ts", "fast": [ + "src/lib/messaging/channels/discord/credential-injection.test.ts", + "src/lib/onboard/messaging-policy-presets.test.ts", + "test/channels/channels-add-preset.test.ts", + "test/e2e/support/hermes-discord-policy-binding.test.ts", + "test/onboarding/onboard-preset-diff.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -400,6 +428,9 @@ { "live": "test/e2e/live/hermes-e2e.test.ts", "fast": [ + "test/e2e/support/managed-image-cohort-contract.test.ts", + "test/e2e/support/managed-image-receipt.test.ts", + "test/e2e/support/stock-managed-image-workflow-boundary.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", "test/e2e/support/inference-adapter.test.ts" @@ -438,6 +469,9 @@ { "live": "test/e2e/live/jetson-nvmap-gpu.test.ts", "fast": [ + "test/e2e/support/managed-image-cohort-contract.test.ts", + "test/e2e/support/managed-image-receipt.test.ts", + "test/e2e/support/stock-managed-image-workflow-boundary.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -452,9 +486,18 @@ { "live": "test/e2e/live/mcp-bridge.test.ts", "fast": [ + "src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts", + "src/lib/actions/sandbox/mcp-bridge-adapter-teardown.test.ts", + "src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts", + "src/lib/actions/sandbox/mcp-bridge-provider.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", - "test/e2e/support/mcp-bridge-tool-discovery.test.ts" + "test/e2e/support/mcp-bridge-hermes-lifecycle.test.ts", + "test/e2e/support/mcp-bridge-onboard-env.test.ts", + "test/e2e/support/mcp-bridge-reliability.test.ts", + "test/e2e/support/mcp-bridge-tool-discovery.test.ts", + "test/e2e/support/mcp-provider-rewrite-probe.test.ts", + "test/e2e/support/mcp-workflow-boundary.test.ts" ] }, { @@ -467,6 +510,13 @@ { "live": "test/e2e/live/messaging-providers.test.ts", "fast": [ + "src/lib/messaging/channels/discord/credential-injection.test.ts", + "src/lib/onboard/credential-provider-registration.test.ts", + "src/lib/onboard/messaging-policy-presets.test.ts", + "src/lib/onboard/providers.test.ts", + "test/channels/channels-add-bridge-lifecycle.test.ts", + "test/e2e/support/messaging-providers-runtime-proofs.test.ts", + "test/runtime/messaging/messaging-build-applier.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -488,6 +538,9 @@ { "live": "test/e2e/live/openclaw-discord-pairing.test.ts", "fast": [ + "src/lib/messaging/channels/discord/credential-injection.test.ts", + "src/lib/onboard/messaging-policy-presets.test.ts", + "test/e2e/support/openclaw-discord-pairing-helpers.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -518,6 +571,9 @@ { "live": "test/e2e/live/openclaw-slack-pairing.test.ts", "fast": [ + "src/lib/onboard/sandbox-create-plan.test.ts", + "test/e2e/support/openclaw-discord-pairing-helpers.test.ts", + "test/onboarding/onboard-messaging.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -633,9 +689,17 @@ }, { "live": "test/e2e/live/channels-stop-start.test.ts", + "liveSources": [ + "test/e2e/live/channels-stop-start-googlechat-proof.ts", + "test/e2e/live/channels-stop-start-helpers.ts", + "test/e2e/live/channels-stop-start-plan-state.ts" + ], "fast": [ + "test/e2e/support/channels-stop-start-cleanup.test.ts", "test/e2e/support/channels-stop-start-config-state.test.ts", - "test/e2e/support/channels-stop-start-googlechat-entry.test.ts", + "test/e2e/support/channels-stop-start-googlechat.test.ts", + "test/e2e/support/channels-stop-start-googlechat-proof.test.ts", + "test/e2e/support/channels-stop-start-plan-state.test.ts", "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", "test/e2e/support/e2e-semantic-phase-check.test.ts", @@ -662,7 +726,15 @@ }, { "live": "test/e2e/live/hermes-slack-e2e.test.ts", - "fast": [ + "liveSources": [ + "test/e2e/live/hermes-slack-credential-transport.ts", + "test/e2e/live/hermes-slack-e2e-helpers.ts", + "test/e2e/live/hermes-slack-proof.ts" + ], + "fast": [ + "test/e2e/support/hermes-slack-cleanup.test.ts", + "test/e2e/support/hermes-slack-credential-transport.test.ts", + "test/e2e/support/hermes-slack-proof.test.ts", "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", "test/e2e/support/e2e-semantic-phase-check.test.ts", @@ -711,10 +783,7 @@ }, { "live": "test/e2e/live/brev-workspace-cleanup.test.ts", - "fast": [ - "test/e2e/support/brev-launchable-fixture.test.ts", - "test/e2e/support/issue-9880-staging-reproduction-workflow.test.ts" - ] + "fast": ["test/e2e/support/brev-launchable-fixture.test.ts"] }, { "live": "test/e2e/live/whatsapp-qr-compact.test.ts", diff --git a/test/e2e/support/channels-stop-start-cleanup.test.ts b/test/e2e/support/channels-stop-start-cleanup.test.ts new file mode 100644 index 00000000000..5ec8081b75d --- /dev/null +++ b/test/e2e/support/channels-stop-start-cleanup.test.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; +import { registerChannelsStopStartProviderCleanup } from "../live/channels-stop-start-helpers.ts"; + +type CleanupAction = { name: string; run: () => Promise | void }; + +function cleanupFixtures(result = { exitCode: 0, stderr: "", stdout: "" }) { + const actions: CleanupAction[] = []; + const cleanup = { + trackDisposable: vi.fn((name: string, run: CleanupAction["run"]) => { + actions.push({ name, run }); + }), + }; + const host = { + command: vi.fn(async () => result), + isCommandAvailable: vi.fn(async () => true), + openshellCommandPath: "openshell", + }; + return { + actions, + cleanup: cleanup as unknown as E2ETargetFixtures["cleanup"], + host: host as unknown as E2ETargetFixtures["host"], + hostMock: host, + trackDisposable: cleanup.trackDisposable, + }; +} + +describe("channels stop/start provider cleanup", () => { + it("registers every exact provider before the live lifecycle starts", () => { + const fixtures = cleanupFixtures(); + + registerChannelsStopStartProviderCleanup(fixtures.cleanup, fixtures.host, { + agent: "openclaw", + env: { NEMOCLAW_SANDBOX_NAME: "e2e-oc-ch-cycle" }, + redactions: ["test-api-key"], + sandboxName: "e2e-oc-ch-cycle", + }); + + expect(fixtures.actions.map(({ name }) => name)).toEqual([ + "delete OpenShell provider e2e-oc-ch-cycle-telegram-bridge", + "delete OpenShell provider e2e-oc-ch-cycle-discord-bridge", + "delete OpenShell provider e2e-oc-ch-cycle-wechat-bridge", + "delete OpenShell provider e2e-oc-ch-cycle-slack-bridge", + "delete OpenShell provider e2e-oc-ch-cycle-slack-app", + "delete OpenShell provider e2e-oc-ch-cycle-teams-bridge", + "delete OpenShell provider e2e-oc-ch-cycle-googlechat-bridge", + ]); + }); + + it("accepts a confirmed absent provider during idempotent cleanup", async () => { + const fixtures = cleanupFixtures({ + exitCode: 1, + stderr: "NotFound: provider does not exist", + stdout: "", + }); + registerChannelsStopStartProviderCleanup(fixtures.cleanup, fixtures.host, { + agent: "hermes", + env: {}, + redactions: [], + sandboxName: "e2e-hm-ch-cycle", + }); + + await expect(fixtures.actions[0]?.run()).resolves.toBeUndefined(); + expect(fixtures.hostMock.command).toHaveBeenCalledWith( + "openshell", + ["provider", "delete", "e2e-hm-ch-cycle-telegram-bridge"], + expect.objectContaining({ + artifactName: + "cleanup-channels-stop-start-openshell-provider-delete-e2e-hm-ch-cycle-telegram-bridge", + }), + ); + }); + + it("rejects an unexpected provider deletion failure", async () => { + const fixtures = cleanupFixtures({ exitCode: 1, stderr: "gateway unavailable", stdout: "" }); + registerChannelsStopStartProviderCleanup(fixtures.cleanup, fixtures.host, { + agent: "openclaw", + env: {}, + redactions: [], + sandboxName: "e2e-oc-ch-cycle", + }); + + await expect(fixtures.actions[0]?.run()).rejects.toThrow( + /cleanup OpenShell provider e2e-oc-ch-cycle-telegram-bridge failed/, + ); + }); + + it("rejects an unsafe sandbox before registering destructive cleanup", () => { + const fixtures = cleanupFixtures(); + + expect(() => + registerChannelsStopStartProviderCleanup(fixtures.cleanup, fixtures.host, { + agent: "openclaw", + env: {}, + redactions: [], + sandboxName: "production-openclaw", + }), + ).toThrow(/only accepts openclaw sandbox names with prefix e2e-oc-ch-/); + expect(fixtures.trackDisposable).not.toHaveBeenCalled(); + }); +}); diff --git a/test/e2e/support/channels-stop-start-googlechat-proof.test.ts b/test/e2e/support/channels-stop-start-googlechat-proof.test.ts new file mode 100644 index 00000000000..f428b9297e4 --- /dev/null +++ b/test/e2e/support/channels-stop-start-googlechat-proof.test.ts @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + assertGooglechatProviderEgressProof, + parseGooglechatProviderEgressProof, +} from "../live/channels-stop-start-googlechat-proof.ts"; + +describe("channels stop/start Google Chat provider-egress proof", () => { + it.each([ + { + agent: "openclaw" as const, + proof: { placeholder: "revision-scoped", statuses: [401] }, + }, + { + agent: "hermes" as const, + proof: { installedOverride: true, placeholder: "revision-scoped", statuses: [401, 401] }, + }, + ])("accepts the $agent provider-egress proof", ({ agent, proof }) => { + const stdout = `runtime noise\n${JSON.stringify(proof)}\n`; + + expect(() => + assertGooglechatProviderEgressProof(parseGooglechatProviderEgressProof(stdout), agent), + ).not.toThrow(); + }); + + it("rejects malformed proof output", () => { + expect(() => parseGooglechatProviderEgressProof("runtime noise\nnot-json\n")).toThrow(); + }); + + it.each([ + { + agent: "openclaw" as const, + proof: { placeholder: "raw-token", statuses: [401] }, + title: "a non-revision-scoped placeholder marker", + }, + { + agent: "openclaw" as const, + proof: { placeholder: "revision-scoped", statuses: [403] }, + title: "an unexpected OpenClaw API status", + }, + { + agent: "hermes" as const, + proof: { installedOverride: true, placeholder: "revision-scoped", statuses: [401] }, + title: "an incomplete Hermes API status sequence", + }, + { + agent: "hermes" as const, + proof: { installedOverride: false, placeholder: "revision-scoped", statuses: [401, 401] }, + title: "a missing Hermes adapter override", + }, + ])("rejects $title", ({ agent, proof }) => { + expect(() => assertGooglechatProviderEgressProof(proof, agent)).toThrow(); + }); +}); diff --git a/test/e2e/support/channels-stop-start-googlechat-entry.test.ts b/test/e2e/support/channels-stop-start-googlechat.test.ts similarity index 88% rename from test/e2e/support/channels-stop-start-googlechat-entry.test.ts rename to test/e2e/support/channels-stop-start-googlechat.test.ts index d3950ba9ff7..a2275cda78d 100644 --- a/test/e2e/support/channels-stop-start-googlechat-entry.test.ts +++ b/test/e2e/support/channels-stop-start-googlechat.test.ts @@ -1,17 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; -import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { addAndRebuildGooglechatForChannelsStopStartLiveE2e, - addGooglechatForChannelsStopStartLiveE2e, GOOGLECHAT_E2E_ACCESS_TOKEN, installGooglechatCredentialFixture, rebuildGooglechatForChannelsStopStartLiveE2e, -} from "../live/channels-stop-start-googlechat-entry.ts"; +} from "../live/channels-stop-start-helpers.ts"; type FixtureRunner = typeof import("../../../src/lib/adapters/openshell/runtime.ts").runOpenshell; type FixtureProviderDependencies = { @@ -27,46 +24,20 @@ type FixtureProviderDependencies = { ): string[]; }; -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); - describe("channels stop/start Google Chat live composition", () => { - it("loads through the standalone live-E2E module boundary (#7317)", () => { - const result = spawnSync( - process.execPath, - [ - "--import", - "tsx", - "-e", - [ - 'import("./test/e2e/live/channels-stop-start-googlechat-entry.ts")', - " .then((module) => console.log(typeof module.addGooglechatForChannelsStopStartLiveE2e))", - " .catch((error) => { console.error(error); process.exitCode = 1; });", - ].join("\n"), - ], - { - cwd: REPO_ROOT, - encoding: "utf8", - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - timeout: 10_000, - }, - ); - - expect(result.status, result.stderr).toBe(0); - expect(result.stdout.trim()).toBe("function"); - }); - it("grants a process-local audience capability to the exact live sandbox", async () => { const addSandboxChannel = vi.fn(async () => {}); + const rebuildSandbox = vi.fn(async () => {}); const restore = vi.fn(); const installCredentialFixture = vi.fn(() => restore); - await addGooglechatForChannelsStopStartLiveE2e( + await addAndRebuildGooglechatForChannelsStopStartLiveE2e( { sandboxName: "e2e-oc-ch-cycle", agent: "openclaw", audience: " https://e2e-fake.trycloudflare.com/googlechat ", }, - { addSandboxChannel, installCredentialFixture }, + { addSandboxChannel, installCredentialFixture, rebuildSandbox }, ); expect(installCredentialFixture).toHaveBeenCalledWith("e2e-oc-ch-cycle", "openclaw"); @@ -79,21 +50,23 @@ describe("channels stop/start Google Chat live composition", () => { }, }, ); + expect(rebuildSandbox).toHaveBeenCalledWith("e2e-oc-ch-cycle", ["--yes"]); expect(restore).toHaveBeenCalledOnce(); }); it("adds Hermes Google Chat without the OpenClaw audience capability", async () => { const addSandboxChannel = vi.fn(async () => {}); + const rebuildSandbox = vi.fn(async () => {}); const restore = vi.fn(); const installCredentialFixture = vi.fn(() => restore); - await addGooglechatForChannelsStopStartLiveE2e( + await addAndRebuildGooglechatForChannelsStopStartLiveE2e( { sandboxName: "e2e-hm-ch-cycle", agent: "hermes", audience: "https://e2e-fake.trycloudflare.com/googlechat", }, - { addSandboxChannel, installCredentialFixture }, + { addSandboxChannel, installCredentialFixture, rebuildSandbox }, ); expect(installCredentialFixture).toHaveBeenCalledWith("e2e-hm-ch-cycle", "hermes"); @@ -102,6 +75,7 @@ describe("channels stop/start Google Chat live composition", () => { { channel: "googlechat" }, {}, ); + expect(rebuildSandbox).toHaveBeenCalledWith("e2e-hm-ch-cycle", ["--yes"]); expect(restore).toHaveBeenCalledOnce(); }); @@ -110,7 +84,7 @@ describe("channels stop/start Google Chat live composition", () => { const installCredentialFixture = vi.fn(() => vi.fn()); await expect( - addGooglechatForChannelsStopStartLiveE2e( + addAndRebuildGooglechatForChannelsStopStartLiveE2e( { sandboxName: "production-openclaw", agent: "openclaw", @@ -128,7 +102,7 @@ describe("channels stop/start Google Chat live composition", () => { const installCredentialFixture = vi.fn(() => vi.fn()); await expect( - addGooglechatForChannelsStopStartLiveE2e( + addAndRebuildGooglechatForChannelsStopStartLiveE2e( { sandboxName: "e2e-oc-ch-cycle", agent: "openclaw", @@ -148,13 +122,17 @@ describe("channels stop/start Google Chat live composition", () => { const restore = vi.fn(); await expect( - addGooglechatForChannelsStopStartLiveE2e( + addAndRebuildGooglechatForChannelsStopStartLiveE2e( { sandboxName: "e2e-hm-ch-cycle", agent: "hermes", audience: "https://e2e-fake.trycloudflare.com/googlechat", }, - { addSandboxChannel, installCredentialFixture: () => restore }, + { + addSandboxChannel, + installCredentialFixture: () => restore, + rebuildSandbox: async () => {}, + }, ), ).rejects.toThrow("planned add failed"); expect(restore).toHaveBeenCalledOnce(); diff --git a/test/e2e/support/channels-stop-start-plan-state.test.ts b/test/e2e/support/channels-stop-start-plan-state.test.ts new file mode 100644 index 00000000000..530acec779f --- /dev/null +++ b/test/e2e/support/channels-stop-start-plan-state.test.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + channelPlanStateErrors, + type ChannelPlanExpectedState, +} from "../live/channels-stop-start-plan-state.ts"; +import type { AgentKind } from "../live/phase6-messaging-helpers.ts"; + +const SANDBOX_NAME = "e2e-channel-cycle"; +const CHANNEL_ID = "slack"; + +function plan(agent: AgentKind, state: ChannelPlanExpectedState): Record { + const present = state !== "removed"; + return { + schemaVersion: 1, + sandboxName: SANDBOX_NAME, + agent, + workflow: + state === "active" + ? "start-channel" + : state === "disabled" + ? "stop-channel" + : "remove-channel", + channels: present + ? [ + { + channelId: CHANNEL_ID, + displayName: "Slack", + authMode: "token-paste", + active: state === "active", + selected: true, + configured: true, + disabled: state === "disabled", + inputs: [], + }, + ] + : [], + disabledChannels: state === "disabled" ? [CHANNEL_ID] : [], + networkPolicy: { + presets: present ? [CHANNEL_ID] : [], + entries: present + ? [ + { + channelId: CHANNEL_ID, + presetName: CHANNEL_ID, + policyKeys: [CHANNEL_ID], + source: "manifest", + }, + ] + : [], + }, + credentialBindings: present + ? [ + { + channelId: CHANNEL_ID, + credentialId: "slackBotToken", + sourceInput: "botToken", + providerName: `${SANDBOX_NAME}-slack-bridge`, + providerEnvKey: "SLACK_BOT_TOKEN", + placeholder: "openshell:resolve:env:SLACK_BOT_TOKEN", + credentialAvailable: true, + }, + ] + : [], + }; +} + +function errors( + value: unknown, + agent: AgentKind = "openclaw", + expected: ChannelPlanExpectedState = "active", +): string[] { + return channelPlanStateErrors(value, { + agent, + channelId: CHANNEL_ID, + credentialBindingRequired: true, + expected, + sandboxName: SANDBOX_NAME, + }); +} + +describe("channels stop/start persisted messaging plan state", () => { + it.each(["openclaw", "hermes"] as const)( + "accepts the complete %s active, disabled, and removed states", + (agent) => { + expect(errors(plan(agent, "active"), agent, "active")).toEqual([]); + expect(errors(plan(agent, "disabled"), agent, "disabled")).toEqual([]); + expect(errors(plan(agent, "removed"), agent, "removed")).toEqual([]); + }, + ); + + it("rejects disabled state that is missing the disabled-channel index", () => { + const value = plan("openclaw", "disabled"); + value.disabledChannels = []; + + expect(errors(value, "openclaw", "disabled")).toEqual([ + "messaging.plan must be a valid persisted plan for e2e-channel-cycle using openclaw", + ]); + }); + + it("rejects a required credential binding that disappeared", () => { + const value = plan("hermes", "active"); + value.credentialBindings = []; + + expect(errors(value, "hermes", "active")).toContain("slack credential binding must be present"); + }); + + it("rejects policy and credential residue after removal", () => { + const value = plan("openclaw", "removed"); + value.networkPolicy = { presets: [CHANNEL_ID], entries: [{ channelId: CHANNEL_ID }] }; + value.credentialBindings = [{ channelId: CHANNEL_ID }]; + + expect(errors(value, "openclaw", "removed")).toEqual([ + "slack policy preset must be removed", + "slack policy entry must be removed", + "slack credential binding must be removed", + ]); + }); + + it("rejects runtime render and hook fields persisted into the plan", () => { + const value = plan("hermes", "active"); + value.agentRender = [ + { + channelId: CHANNEL_ID, + agent: "hermes", + target: "config.yaml", + kind: "json-fragment", + path: "platforms.slack", + value: { enabled: true }, + templateRefs: [], + }, + ]; + value.channels = (value.channels as Record[]).map((channel) => ({ + ...channel, + hooks: [], + })); + + expect(errors(value, "hermes", "active")).toEqual([ + "messaging.plan.agentRender must not persist", + "messaging.plan channel hooks must not persist", + ]); + }); +}); diff --git a/test/e2e/support/hermes-slack-cleanup.test.ts b/test/e2e/support/hermes-slack-cleanup.test.ts new file mode 100644 index 00000000000..5a35b5d3333 --- /dev/null +++ b/test/e2e/support/hermes-slack-cleanup.test.ts @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; +import { registerHermesSlackCleanup } from "../live/hermes-slack-e2e-helpers.ts"; + +type CleanupAction = { name: string; run: () => Promise | void }; + +function cleanupFixtures(result = { exitCode: 0, stderr: "", stdout: "" }) { + const actions: CleanupAction[] = []; + const cleanup = { + trackDisposable: vi.fn((name: string, run: CleanupAction["run"]) => { + actions.push({ name, run }); + }), + trackGateway: vi.fn(), + trackSandbox: vi.fn(), + }; + const host = { + command: vi.fn(async () => result), + isCommandAvailable: vi.fn(async () => true), + openshellCommandPath: "openshell", + }; + return { + actions, + cleanup, + fixtures: { + cleanup: cleanup as unknown as E2ETargetFixtures["cleanup"], + host: host as unknown as E2ETargetFixtures["host"], + sandbox: {} as E2ETargetFixtures["sandbox"], + }, + host, + }; +} + +describe("Hermes Slack retained-resource cleanup", () => { + it("does not register destructive cleanup when sandbox retention is requested", () => { + const { cleanup, fixtures } = cleanupFixtures(); + + registerHermesSlackCleanup(fixtures, { + apiKey: "test-api-key", + env: {}, + keepSandbox: true, + redactionValues: ["test-api-key"], + sandboxName: "e2e-hermes-slack", + }); + + expect(cleanup.trackGateway).not.toHaveBeenCalled(); + expect(cleanup.trackDisposable).not.toHaveBeenCalled(); + expect(cleanup.trackSandbox).not.toHaveBeenCalled(); + }); + + it("registers gateway, provider, and sandbox cleanup by default", () => { + const { cleanup, fixtures } = cleanupFixtures(); + + registerHermesSlackCleanup(fixtures, { + apiKey: "test-api-key", + env: {}, + keepSandbox: false, + redactionValues: ["test-api-key"], + sandboxName: "e2e-hermes-slack", + }); + + expect(cleanup.trackGateway).toHaveBeenCalledTimes(1); + expect(cleanup.trackDisposable).toHaveBeenCalledTimes(3); + expect(cleanup.trackSandbox).toHaveBeenCalledTimes(1); + }); + + it("accepts a confirmed absent provider during idempotent cleanup", async () => { + const { actions, fixtures, host } = cleanupFixtures({ + exitCode: 1, + stderr: "No provider named e2e-hermes-slack-slack-app", + stdout: "", + }); + registerHermesSlackCleanup(fixtures, { + apiKey: "test-api-key", + env: {}, + keepSandbox: false, + redactionValues: ["test-api-key"], + sandboxName: "e2e-hermes-slack", + }); + + await actions[0]?.run(); + expect(host.command).toHaveBeenCalledWith( + "openshell", + ["provider", "delete", "e2e-hermes-slack-slack-app"], + expect.objectContaining({ + artifactName: "cleanup-hermes-slack-openshell-provider-delete-e2e-hermes-slack-slack-app", + }), + ); + }); + + it("rejects an unexpected provider deletion failure", async () => { + const { actions, fixtures } = cleanupFixtures({ + exitCode: 1, + stderr: "gateway unavailable", + stdout: "", + }); + registerHermesSlackCleanup(fixtures, { + apiKey: "test-api-key", + env: {}, + keepSandbox: false, + redactionValues: ["test-api-key"], + sandboxName: "e2e-hermes-slack", + }); + + await expect(actions[0]?.run()).rejects.toThrow( + /cleanup OpenShell provider e2e-hermes-slack-slack-app failed/, + ); + }); + + it("rejects a non-E2E sandbox name before registering destructive cleanup", () => { + const { cleanup, fixtures } = cleanupFixtures(); + + expect(() => + registerHermesSlackCleanup(fixtures, { + apiKey: "test-api-key", + env: {}, + keepSandbox: false, + redactionValues: ["test-api-key"], + sandboxName: "shared-hermes-sandbox", + }), + ).toThrow( + "Hermes Slack live test is destructive and only accepts sandbox name e2e-hermes-slack; got shared-hermes-sandbox", + ); + + expect(cleanup.trackGateway).not.toHaveBeenCalled(); + expect(cleanup.trackDisposable).not.toHaveBeenCalled(); + expect(cleanup.trackSandbox).not.toHaveBeenCalled(); + }); +}); diff --git a/test/e2e/support/hermes-slack-credential-transport.test.ts b/test/e2e/support/hermes-slack-credential-transport.test.ts new file mode 100644 index 00000000000..36181e1c9e9 --- /dev/null +++ b/test/e2e/support/hermes-slack-credential-transport.test.ts @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + HERMES_SLACK_CREDENTIAL_FINGERPRINT_SCAN_SOURCE, + hermesSlackCredentialFingerprints, + hermesSlackCredentialScanScript, +} from "../live/hermes-slack-credential-transport.ts"; +import { assertHermesSlackCredentialFingerprintScanResult } from "../live/hermes-slack-e2e-helpers.ts"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { force: true, recursive: true }); + } +}); + +function runCredentialTransport(transportFailure = false) { + const root = mkdtempSync(join(tmpdir(), "nemoclaw-hermes-slack-exec-")); + temporaryDirectories.push(root); + const openshell = join(root, "openshell"); + const calls = join(root, "openshell-calls.log"); + const ssh = join(root, "ssh"); + + writeFileSync( + openshell, + [ + "#!/bin/sh", + 'printf "args=%s\\n" "$*" >>"$OPENSHELL_CALLS"', + 'payload="$(cat)"', + 'printf "payload=%s\\n" "$payload" >>"$OPENSHELL_CALLS"', + '[ "${TRANSPORT_FAILURE:-0}" = 1 ] && exit 70', + 'printf "OK\\n"', + ].join("\n"), + ); + chmodSync(openshell, 0o755); + writeFileSync( + ssh, + ["#!/bin/sh", 'printf "ssh-args=%s\\n" "$*" >>"$OPENSHELL_CALLS"', "exit 71"].join("\n"), + ); + chmodSync(ssh, 0o755); + + const credentials = ["xoxb-test-credential", "xapp-test-credential"]; + const credentialFingerprints = hermesSlackCredentialFingerprints(credentials); + const script = hermesSlackCredentialScanScript({ + credentialFingerprints, + openshellCommandPath: openshell, + remoteCommand: "cat >/dev/null", + sandboxName: "e2e-hermes-slack", + }); + const result = spawnSync("bash", ["-c", script], { + encoding: "utf8", + env: { + OPENSHELL_CALLS: calls, + PATH: `${root}:${process.env.PATH ?? ""}`, + TRANSPORT_FAILURE: transportFailure ? "1" : "0", + }, + timeout: 5_000, + }); + return { + calls: readFileSync(calls, "utf8"), + credentialFingerprints, + credentials, + result, + }; +} + +describe("Hermes Slack credential-fingerprint scan", () => { + it("sends only derived fingerprints through the OpenShell sandbox exec boundary", () => { + const { calls, credentialFingerprints, credentials, result } = runCredentialTransport(); + + expect(result.status, result.stderr).toBe(0); + expect(calls).toContain("args=sandbox exec --name e2e-hermes-slack -- sh -lc cat >/dev/null"); + const payload = calls.match(/^payload=(.+)$/mu)?.[1]; + expect(JSON.parse(payload ?? "null")).toEqual(credentialFingerprints); + expect(calls).not.toContain(credentials[0]); + expect(calls).not.toContain(credentials[1]); + expect(calls).not.toContain("ssh-args="); + }); + + it("detects an exact credential substring without receiving the credential", () => { + const root = mkdtempSync(join(tmpdir(), "nemoclaw-hermes-slack-fingerprint-")); + temporaryDirectories.push(root); + const fixture = join(root, "fixture.log"); + const credential = "xoxb-test-fingerprint-only"; + const input = JSON.stringify(hermesSlackCredentialFingerprints([credential])); + const scan = () => + spawnSync( + "python3", + ["-c", HERMES_SLACK_CREDENTIAL_FINGERPRINT_SCAN_SOURCE, JSON.stringify([fixture]), "files"], + { encoding: "utf8", input }, + ); + + writeFileSync(fixture, `prefix ${credential} suffix`); + const leaked = scan(); + expect(leaked.status, leaked.stderr).toBe(0); + expect(JSON.parse(leaked.stdout)).toEqual({ files: "LEAK", processes: "EMPTY" }); + + writeFileSync(fixture, "only revision-scoped placeholders remain"); + const clean = scan(); + expect(clean.status, clean.stderr).toBe(0); + expect(JSON.parse(clean.stdout)).toEqual({ files: "OK", processes: "EMPTY" }); + }); + + it("propagates an OpenShell sandbox exec transport failure", () => { + const { calls, result } = runCredentialTransport(true); + + expect(result.status).not.toBe(0); + expect(calls).toContain("args=sandbox exec --name e2e-hermes-slack"); + }); + + it("rejects missing process-argument evidence", () => { + expect(() => + assertHermesSlackCredentialFingerprintScanResult({ files: "OK", processes: "EMPTY" }), + ).toThrow(/raw Slack token absent from process arguments/); + }); +}); diff --git a/test/e2e/support/hermes-slack-proof.test.ts b/test/e2e/support/hermes-slack-proof.test.ts new file mode 100644 index 00000000000..f2373b27cf7 --- /dev/null +++ b/test/e2e/support/hermes-slack-proof.test.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + assertHermesSlackApiProof, + classifyHermesSlackApiProof, +} from "../live/hermes-slack-proof.ts"; + +describe("Hermes Slack API proof classification", () => { + it("accepts both successful Slack API markers", () => { + expect( + classifyHermesSlackApiProof( + "OK auth.test: status=200 error=None\nOK apps.connections.open: status=200 error=None\n", + ), + ).toEqual({ kind: "passed" }); + }); + + it("rejects a provider timeout as incomplete required evidence", () => { + expect(classifyHermesSlackApiProof("TIMEOUT auth.test: socket timeout\n")).toEqual({ + kind: "timeout", + reason: "TIMEOUT auth.test: socket timeout", + }); + expect(() => assertHermesSlackApiProof("TIMEOUT auth.test: socket timeout\n")).toThrow( + "Slack API proof incomplete required evidence: TIMEOUT auth.test: socket timeout", + ); + }); + + it("accepts complete required evidence in the live assertion path", () => { + expect(() => + assertHermesSlackApiProof( + "OK auth.test: status=200 error=None\nOK apps.connections.open: status=200 error=None\n", + ), + ).not.toThrow(); + }); + + it.each(["FAIL auth.test: status=403", "ERROR apps.connections.open: invalid response"])( + "rejects an explicit probe failure: %s", + (line) => { + expect(classifyHermesSlackApiProof(`${line}\n`)).toEqual({ + kind: "failed", + reason: line, + }); + }, + ); + + it.each([ + { + output: "OK auth.test: status=200 error=None\n", + reason: "missing successful probe marker: apps.connections.open", + }, + { + output: "OK apps.connections.open: status=200 error=None\n", + reason: "missing successful probe marker: auth.test", + }, + { + output: "unstructured output\n", + reason: "missing successful probe marker: auth.test, apps.connections.open", + }, + ])("rejects incomplete success evidence", ({ output, reason }) => { + expect(classifyHermesSlackApiProof(output)).toEqual({ kind: "failed", reason }); + }); +}); diff --git a/test/e2e/support/issue-9880-staging-reproduction-workflow.test.ts b/test/e2e/support/issue-9880-staging-reproduction-workflow.test.ts deleted file mode 100644 index 8e2183946ac..00000000000 --- a/test/e2e/support/issue-9880-staging-reproduction-workflow.test.ts +++ /dev/null @@ -1,418 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - ISSUE_9880_STAGING_LAUNCHABLE_CLEANUP_TIMEOUT_MS, - ISSUE_9880_STAGING_LAUNCHABLE_ONBOARD_TIMEOUT_MS, - ISSUE_9880_STAGING_LAUNCHABLE_SCENARIO_TIMEOUT_MS, - ISSUE_9880_STAGING_LAUNCHABLE_TEST_TIMEOUT_MS, -} from "../../../tools/e2e/staging-launchable-timeout-contract.mts"; -import { - DEFAULT_BREV_EXEC_READY_TIMEOUT_MS, - DEFAULT_BREV_IDENTITY_TIMEOUT_MS, - DEFAULT_BREV_STAGING_HANDOFF_TIMEOUT_MS, - DEFAULT_BREV_WORKSPACE_CREATE_TIMEOUT_MS, - DEFAULT_BREV_WORKSPACE_DELETE_TIMEOUT_MS, - DEFAULT_BREV_WORKSPACE_READY_TIMEOUT_MS, -} from "../fixtures/brev-launchable.ts"; -import { - readYaml, - type Workflow, - type WorkflowJob, - type WorkflowStep, -} from "../../helpers/e2e-workflow-contract.ts"; - -const MINUTE_MS = 60_000; -const CHECKOUT_ACTION = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; -const UPLOAD_ACTION = - "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57"; - -type StagingWorkflow = Workflow & { - concurrency?: { group?: string; "cancel-in-progress"?: boolean }; - permissions?: Record; -}; - -type WorkflowContract = { - checkout: string; - inferenceCredential: string; - job: string; - prepare: string; - scenario: string; - upload: string; - workflow: string; -}; - -type TimeoutContract = { - cleanupTimeoutMs: number; - testTimeoutMs: number; -}; - -const contract = { - workflow: ".github/workflows/issue-9880-staging-reproduction.yaml", - job: "reproduce", - checkout: "Check out trusted reproduction lane", - prepare: "Prepare Brev CLI and evidence directory", - scenario: "Reproduce issue 9880 on the staging Launchable", - upload: "Upload issue 9880 evidence", - inferenceCredential: "NVIDIA_API_KEY", -} as const satisfies WorkflowContract; - -const timeoutContract: TimeoutContract = { - cleanupTimeoutMs: ISSUE_9880_STAGING_LAUNCHABLE_CLEANUP_TIMEOUT_MS, - testTimeoutMs: ISSUE_9880_STAGING_LAUNCHABLE_TEST_TIMEOUT_MS, -}; - -const PREPARATION_STEP_NAMES = [ - contract.checkout, - "Authorize maintainer dispatch", - "Set up Node", - "Install dependencies", - contract.prepare, -] as const; -const POST_SCENARIO_STEP_NAMES = [ - "Verify workflow-owned workspace cleanup", - "Remove Brev credentials", - contract.upload, -] as const; - -const CONTROLLER_OPERATION_TIMEOUT_MS = - DEFAULT_BREV_STAGING_HANDOFF_TIMEOUT_MS + - DEFAULT_BREV_WORKSPACE_CREATE_TIMEOUT_MS + - DEFAULT_BREV_WORKSPACE_READY_TIMEOUT_MS + - DEFAULT_BREV_EXEC_READY_TIMEOUT_MS + - DEFAULT_BREV_IDENTITY_TIMEOUT_MS + - ISSUE_9880_STAGING_LAUNCHABLE_ONBOARD_TIMEOUT_MS + - ISSUE_9880_STAGING_LAUNCHABLE_SCENARIO_TIMEOUT_MS; - -const mutations: ReadonlyArray< - readonly [ - string, - (workflow: StagingWorkflow, contract: WorkflowContract, timeouts: TimeoutContract) => void, - string, - ] -> = [ - [ - "an untrusted checkout reference", - (workflow, contract) => { - step(workflow, contract, contract.checkout).with!.ref = "${{ github.sha }}"; - }, - "checkout must use trusted workflow code without persisted credentials", - ], - [ - "missing maintainer authorization", - (workflow, contract) => { - step(workflow, contract, "Authorize maintainer dispatch").run = "true"; - }, - "workflow must authorize both dispatch actors as maintainers", - ], - [ - "widened workflow permission", - (workflow) => { - workflow.permissions = { contents: "write" }; - }, - "workflow permissions must remain read-only", - ], - [ - "missing Brev CLI checksum verification", - (workflow, contract) => { - const prepare = step(workflow, contract, contract.prepare); - prepare.run = String(prepare.run).replace("sha256sum -c -", "true"); - }, - "Brev CLI download must retain checksum verification", - ], - [ - "conditional workspace cleanup", - (workflow, contract) => { - step(workflow, contract, "Verify workflow-owned workspace cleanup").if = "success()"; - }, - "workflow must always reconcile its owned Brev workspace before removing credentials", - ], - [ - "conditional credential cleanup", - (workflow, contract) => { - step(workflow, contract, "Remove Brev credentials").if = "success()"; - }, - "workflow must always remove and verify its Brev credential directory", - ], - [ - "a test timeout shorter than the controller lifecycle", - (_workflow, _contract, timeouts) => { - timeouts.testTimeoutMs = CONTROLLER_OPERATION_TIMEOUT_MS - 1; - }, - "test timeout must contain every sequential controller operation", - ], - [ - "a cleanup timeout that cannot contain Brev deletion", - (_workflow, _contract, timeouts) => { - timeouts.cleanupTimeoutMs = DEFAULT_BREV_WORKSPACE_DELETE_TIMEOUT_MS; - }, - "cleanup timeout must exceed the Brev deletion budget", - ], - [ - "a scenario step timeout without cleanup time", - (workflow, contract, timeouts) => { - step(workflow, contract, contract.scenario)["timeout-minutes"] = - (timeouts.testTimeoutMs + timeouts.cleanupTimeoutMs) / MINUTE_MS - 1; - }, - "scenario timeout must contain the live test and cleanup budgets", - ], - [ - "an unbounded preparation step", - (workflow, contract) => { - delete step(workflow, contract, contract.prepare)["timeout-minutes"]; - }, - "workflow preparation and finalization steps must have positive timeouts", - ], - [ - "a job timeout without preparation and finalization time", - (workflow, contract, timeouts) => { - const job = workflow.jobs[contract.job]!; - const reservedStepMs = [...PREPARATION_STEP_NAMES, ...POST_SCENARIO_STEP_NAMES].reduce( - (total, name) => - total + Number(step(workflow, contract, name)["timeout-minutes"]) * MINUTE_MS, - 0, - ); - job["timeout-minutes"] = - (reservedStepMs + timeouts.testTimeoutMs + timeouts.cleanupTimeoutMs) / MINUTE_MS - 1; - }, - "job timeout must contain preparation, scenario, cleanup, and finalization budgets", - ], -]; - -describe("rejects unsafe changes to the staging Launchable workflow for issue 9880", () => { - it.each(mutations)("rejects %s", (_case, mutate, expected) => { - const workflow = readStagingWorkflow(); - const timeouts = structuredClone(timeoutContract); - expect(validateWorkflow(workflow, timeouts)).not.toContain(expected); - mutate(workflow, contract, timeouts); - - expect(validateWorkflow(workflow, timeouts)).toContain(expected); - }); -}); - -function readStagingWorkflow(): StagingWorkflow { - return structuredClone(readYaml(contract.workflow)); -} - -function step( - workflow: StagingWorkflow, - contract: WorkflowContract, - name: string, -): WorkflowStep & { "timeout-minutes"?: number } { - return workflow.jobs[contract.job]!.steps!.find((entry) => entry.name === name)!; -} - -function validateWorkflow(workflow: StagingWorkflow, timeouts: TimeoutContract): string[] { - const errors: string[] = []; - const job = workflow.jobs[contract.job] ?? {}; - - recordValidation( - errors, - timeouts.testTimeoutMs >= CONTROLLER_OPERATION_TIMEOUT_MS, - "test timeout must contain every sequential controller operation", - ); - recordValidation( - errors, - timeouts.cleanupTimeoutMs > DEFAULT_BREV_WORKSPACE_DELETE_TIMEOUT_MS, - "cleanup timeout must exceed the Brev deletion budget", - ); - - recordValidation( - errors, - JSON.stringify(workflow.permissions) === JSON.stringify({ contents: "read" }), - "workflow permissions must remain read-only", - ); - recordValidation( - errors, - workflow.concurrency?.["cancel-in-progress"] === false && - workflow.concurrency.group === "issue-9880-staging-launchable", - "workflow must retain its non-cancelling staging concurrency group", - ); - recordValidation( - errors, - job.if === - "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' }}", - "workflow job must remain manual and trusted-main-only", - ); - recordValidation( - errors, - job["runs-on"] === "ubuntu-latest", - "workflow job must remain on a GitHub-hosted runner", - ); - - validateCheckout(errors, job, contract); - validateAuthorization(errors, job); - validatePreparation(errors, job, contract); - validateScenario(errors, job, contract, timeouts); - validateWorkspaceCleanup(errors, job, contract, timeouts); - validateCredentialCleanup(errors, job, contract); - validateEvidenceUpload(errors, job, contract); - return errors; -} - -function validateCheckout(errors: string[], job: WorkflowJob, contract: WorkflowContract): void { - const checkout = namedStep(job, contract.checkout); - recordValidation( - errors, - checkout?.uses === CHECKOUT_ACTION && - checkout.env === undefined && - checkout.with?.ref === "${{ github.workflow_sha }}" && - checkout.with?.["persist-credentials"] === false, - "checkout must use trusted workflow code without persisted credentials", - ); -} - -function validateAuthorization(errors: string[], job: WorkflowJob): void { - const authorize = namedStep(job, "Authorize maintainer dispatch"); - const run = String(authorize?.run ?? ""); - recordValidation( - errors, - authorize?.env?.ACTOR === "${{ github.actor }}" && - authorize.env.GITHUB_TOKEN === "${{ github.token }}" && - authorize.env.TRIGGERING_ACTOR === "${{ github.triggering_actor }}" && - run.includes('for actor in "$ACTOR" "$TRIGGERING_ACTOR"') && - run.includes("maintain|admin") && - run.includes("/collaborators/${actor}/permission"), - "workflow must authorize both dispatch actors as maintainers", - ); -} - -function validatePreparation(errors: string[], job: WorkflowJob, contract: WorkflowContract): void { - const prepare = namedStep(job, contract.prepare); - const run = String(prepare?.run ?? ""); - recordValidation( - errors, - /^[0-9a-f]{64}$/u.test(String(prepare?.env?.BREV_CLI_SHA256 ?? "")) && - /^\d+[.]\d+[.]\d+$/u.test(String(prepare?.env?.BREV_CLI_VERSION ?? "")) && - run.includes("sha256sum -c -") && - run.includes('brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID"'), - "Brev CLI download must retain checksum verification", - ); - recordValidation( - errors, - prepare?.env?.BREV_API_KEY === "${{ secrets.BREV_API_KEY }}" && - prepare.env.BREV_ORG_ID === "${{ secrets.BREV_ORG_ID }}" && - prepare.env[contract.inferenceCredential] === undefined && - prepare.env.NEMOCLAW_IMAGE_DISPATCH_TOKEN === undefined, - "workflow credentials must remain scoped to their owning steps", - ); -} - -function validateScenario( - errors: string[], - job: WorkflowJob, - contract: WorkflowContract, - timeouts: TimeoutContract, -): void { - const scenario = namedStep(job, contract.scenario) as - | (WorkflowStep & { "timeout-minutes"?: number }) - | undefined; - const scenarioTimeoutMs = Number(scenario?.["timeout-minutes"]) * MINUTE_MS; - recordValidation( - errors, - Number.isSafeInteger(scenarioTimeoutMs) && - scenarioTimeoutMs >= timeouts.testTimeoutMs + timeouts.cleanupTimeoutMs, - "scenario timeout must contain the live test and cleanup budgets", - ); - const boundedWorkflowStepNames = [...PREPARATION_STEP_NAMES, ...POST_SCENARIO_STEP_NAMES]; - const boundedWorkflowSteps = boundedWorkflowStepNames.map( - (name) => namedStep(job, name) as (WorkflowStep & { "timeout-minutes"?: number }) | undefined, - ); - recordValidation( - errors, - boundedWorkflowSteps.every((workflowStep) => Number(workflowStep?.["timeout-minutes"]) > 0), - "workflow preparation and finalization steps must have positive timeouts", - ); - const reservedWorkflowStepMs = boundedWorkflowSteps.reduce( - (total, workflowStep) => total + Number(workflowStep?.["timeout-minutes"] ?? 0) * MINUTE_MS, - 0, - ); - const jobTimeoutMs = Number(job["timeout-minutes"]) * MINUTE_MS; - recordValidation( - errors, - Number.isSafeInteger(jobTimeoutMs) && - jobTimeoutMs >= reservedWorkflowStepMs + timeouts.testTimeoutMs + timeouts.cleanupTimeoutMs, - "job timeout must contain preparation, scenario, cleanup, and finalization budgets", - ); - recordValidation( - errors, - scenario?.env?.BREV_LAUNCHABLE_ID === "${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}" && - scenario.env.NEMOCLAW_IMAGE_DISPATCH_TOKEN === - "${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}" && - scenario.env[contract.inferenceCredential] === - `\${{ secrets.${contract.inferenceCredential} }}` && - scenario.env.BREV_API_KEY === undefined && - scenario.env.BREV_ORG_ID === undefined && - scenario.env.PATH === "/usr/local/bin:/usr/bin:/bin", - "workflow credentials must remain scoped to their owning steps", - ); -} - -function validateWorkspaceCleanup( - errors: string[], - job: WorkflowJob, - contract: WorkflowContract, - timeouts: TimeoutContract, -): void { - const prepare = namedStep(job, contract.prepare); - const scenario = namedStep(job, contract.scenario); - const cleanup = namedStep(job, "Verify workflow-owned workspace cleanup") as - | (WorkflowStep & { "timeout-minutes"?: number }) - | undefined; - const cleanupCommand = String(cleanup?.run).trim(); - recordValidation( - errors, - cleanup?.if === "${{ always() && steps.prepare.outputs.work_dir != '' }}" && - cleanup.env?.BREV_WORKSPACE_OWNERSHIP_FILE === - scenario?.env?.BREV_WORKSPACE_OWNERSHIP_FILE && - cleanup.env?.HOME === prepare?.env?.HOME && - cleanup.env?.NEMOCLAW_RUN_LIVE_E2E === "1" && - cleanup.env?.PATH === "/usr/local/bin:/usr/bin:/bin" && - cleanupCommand === - "./node_modules/.bin/vitest run --project e2e-live test/e2e/live/brev-workspace-cleanup.test.ts --silent=false --reporter=default" && - Number(cleanup["timeout-minutes"]) * MINUTE_MS >= timeouts.cleanupTimeoutMs, - "workflow must always reconcile its owned Brev workspace before removing credentials", - ); -} - -function validateCredentialCleanup( - errors: string[], - job: WorkflowJob, - contract: WorkflowContract, -): void { - const prepare = namedStep(job, contract.prepare); - const cleanup = namedStep(job, "Remove Brev credentials"); - const run = String(cleanup?.run ?? ""); - recordValidation( - errors, - cleanup?.if === "always()" && - cleanup.env?.HOME === prepare?.env?.HOME && - run.includes('rm -rf -- "$HOME"') && - run.includes('test ! -e "$HOME"'), - "workflow must always remove and verify its Brev credential directory", - ); -} - -function validateEvidenceUpload( - errors: string[], - job: WorkflowJob, - contract: WorkflowContract, -): void { - const upload = namedStep(job, contract.upload); - recordValidation( - errors, - upload?.uses === UPLOAD_ACTION && - upload.if === "${{ always() && steps.prepare.outputs.work_dir != '' }}", - "workflow must always upload evidence through the pinned repository action", - ); -} - -function namedStep(job: WorkflowJob, name: string): WorkflowStep | undefined { - return job.steps?.find((entry) => entry.name === name); -} - -function recordValidation(errors: string[], valid: boolean, message: string): void { - errors.push(...(valid ? [] : [message])); -} diff --git a/test/helpers/onboard-child-process-harness.ts b/test/helpers/onboard-child-process-harness.ts index 15452e274b8..42a8d21bf96 100644 --- a/test/helpers/onboard-child-process-harness.ts +++ b/test/helpers/onboard-child-process-harness.ts @@ -73,6 +73,10 @@ export interface RunOnboardProcessOptions { cwd?: string; /** Kill the child after this many milliseconds. */ timeoutMs?: number; + /** Signal used when the timeout expires. */ + killSignal?: NodeJS.Signals; + /** Optional stdin for interactive process fixtures. */ + input?: string; } /** The decoded outcome of one spawned process run. */ @@ -96,6 +100,8 @@ export function runOnboardProcess( encoding: "utf-8", env: options.env, ...(options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs }), + ...(options.killSignal === undefined ? {} : { killSignal: options.killSignal }), + ...(options.input === undefined ? {} : { input: options.input }), }); const stdout = result.stdout ?? ""; const stderr = result.stderr ?? ""; @@ -109,6 +115,14 @@ export function runOnboardProcess( }; } +/** Runs a generated onboarding script with a bounded hard-kill timeout. */ +export function runBoundedOnboardScript( + scriptPath: string, + options: Omit, +): OnboardProcessResult { + return runOnboardProcess([scriptPath], { ...options, timeoutMs: 45_000, killSignal: "SIGKILL" }); +} + /** * Parses the last stdout line that is a JSON object; scenario scripts print * their result payload after any incidental logging. Throws with the full diff --git a/test/helpers/onboard-fixture-contract.json b/test/helpers/onboard-fixture-contract.json new file mode 100644 index 00000000000..286d91d9353 --- /dev/null +++ b/test/helpers/onboard-fixture-contract.json @@ -0,0 +1,3 @@ +{ + "createdSandboxId": "sbx-4f2a91c0d7" +} diff --git a/test/helpers/onboard-openshell-fixture.ts b/test/helpers/onboard-openshell-fixture.ts index 6494e383f8e..f216c996c7e 100644 --- a/test/helpers/onboard-openshell-fixture.ts +++ b/test/helpers/onboard-openshell-fixture.ts @@ -4,6 +4,10 @@ import fs from "node:fs"; import path from "node:path"; +import onboardFixtureContract from "./onboard-fixture-contract.json"; + +export const ONBOARD_CREATED_SANDBOX_ID = onboardFixtureContract.createdSandboxId; + function writeExecutable(target: string, contents: string): void { fs.writeFileSync(target, contents, { mode: 0o755 }); } @@ -14,7 +18,7 @@ export function writeOkOpenshell( ): void { const gatewayPort = options.gatewayPort ?? 8080; const sandboxGet = options.readySandboxGet - ? 'if [ "${1:-}" = sandbox ] && [ "${2:-}" = get ]; then printf "Sandbox:\\n\\n Id: sbx-4f2a91c0d7\\n Name: %s\\n Phase: Ready\\n" "${!#}"; fi\n' + ? `if [ "\${1:-}" = sandbox ] && [ "\${2:-}" = get ]; then printf "Sandbox:\\n\\n Id: ${ONBOARD_CREATED_SANDBOX_ID}\\n Name: %s\\n Phase: Ready\\n" "\${!#}"; fi\n` : ""; writeExecutable( path.join(fakeBin, "openshell"), diff --git a/test/helpers/onboard-script-mocks-policy-authority.test.ts b/test/helpers/onboard-script-mocks-policy-authority.test.ts index 1366ed85896..04118e677f2 100644 --- a/test/helpers/onboard-script-mocks-policy-authority.test.ts +++ b/test/helpers/onboard-script-mocks-policy-authority.test.ts @@ -7,6 +7,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { CaptureOpenshellResult } from "../../src/lib/adapters/openshell/client"; import { + ONBOARD_CREATED_SANDBOX_ID, mockCreatedSandboxIdentityList, mockStructuredOpenShellCaptureFromRunner, } from "./onboard-script-mocks.cjs"; @@ -164,7 +165,7 @@ describe("mockStructuredOpenShellCaptureFromRunner", () => { ["sandbox", "get", "-g", "nemoclaw-test", "my-assistant"], { includeStreams: true }, ).stdout, - ).toContain("Id: sbx-4f2a91c0d7"); + ).toContain(`Id: ${ONBOARD_CREATED_SANDBOX_ID}`); }); it.each([ diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index e400991a002..096b945c287 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -254,7 +254,9 @@ function createStatefulMessagingProviderRunner({ ) { return { status: 0, - stdout: Buffer.from(`Name: ${readySandboxName}\nId: sbx-4f2a91c0d7\nPhase: Ready\n`), + stdout: Buffer.from( + `Name: ${readySandboxName}\nId: ${ONBOARD_CREATED_SANDBOX_ID}\nPhase: Ready\n`, + ), stderr: Buffer.alloc(0), }; } @@ -289,6 +291,7 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [ const ONBOARD_SANDBOX_OLD_CONTAINER_ID = "a".repeat(64); const ONBOARD_SANDBOX_NEW_CONTAINER_ID = "b".repeat(64); +const { createdSandboxId: ONBOARD_CREATED_SANDBOX_ID } = require("./onboard-fixture-contract.json"); const ONBOARD_SANDBOX_INSPECT = { Id: ONBOARD_SANDBOX_OLD_CONTAINER_ID, Image: `sha256:${"c".repeat(64)}`, @@ -415,7 +418,8 @@ function mockCreatedSandboxIdentityList(command, options = {}) { args[2] !== "-g" || args[3] !== gatewayName || args[4] !== "--selector" || - !new RegExp(`^${prefix}[0-9a-f]{62}$`, "u").test(selector) || + !selector.startsWith(prefix) || + !/^[0-9a-f]{62}$/u.test(selector.slice(prefix.length)) || args[6] !== "--output" || args[7] !== "json" || args[8] !== "--limit" || @@ -426,7 +430,7 @@ function mockCreatedSandboxIdentityList(command, options = {}) { const nonce = selector.slice(prefix.length); publishedCreatedGatewayName = gatewayName; publishedCreatedSandboxIdentity = { - id: options.sandboxId || "sbx-4f2a91c0d7", + id: options.sandboxId || ONBOARD_CREATED_SANDBOX_ID, name: options.sandboxName || "my-assistant", labels: { "ai.nvidia.nemoclaw.create-attempt": nonce }, resource_version: 1, @@ -723,7 +727,7 @@ function managedSandboxPolicyReceiptFixture(entry, options = {}) { const gatewayName = options.gatewayName || "nemoclaw"; const gatewayPort = options.gatewayPort || 8080; const lifecycleGeneration = options.lifecycleGeneration || "123e4567-e89b-42d3-a456-426614174983"; - const sandboxId = options.sandboxId || "sbx-4f2a91c0d7"; + const sandboxId = options.sandboxId || ONBOARD_CREATED_SANDBOX_ID; const sandboxIdentityFingerprint = require("node:crypto") .createHash("sha256") .update(sandboxId) @@ -1029,7 +1033,7 @@ function mockManagedImageBootstrap() { path.resolve(__dirname, "../../src/lib/adapters/openshell/sandbox-identity.ts"), ); - sandboxIdentity.resolveOpenShellSandboxId = () => "sbx-4f2a91c0d7"; + sandboxIdentity.resolveOpenShellSandboxId = () => ONBOARD_CREATED_SANDBOX_ID; authorityStore.createDockerManagedBootstrapAuthorityStore = () => ({ async recordPreparedAuthority(authority) { return { @@ -1178,6 +1182,7 @@ if (process.env.NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG === "1") { } module.exports = { + ONBOARD_CREATED_SANDBOX_ID, mockEndpointlessProviderProfileRun, mockManagedEndpointlessProviderProfileRun, createStatefulMessagingProviderRunner, diff --git a/test/onboarding/onboard-custom-dockerfile.test.ts b/test/onboarding/onboard-custom-dockerfile.test.ts index 129bff69121..2e6c5fb4d91 100644 --- a/test/onboarding/onboard-custom-dockerfile.test.ts +++ b/test/onboarding/onboard-custom-dockerfile.test.ts @@ -235,7 +235,7 @@ runner.run = (command, opts = {}) => { if (profileResult !== null) return profileResult; if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { diff --git a/test/onboarding/onboard-extra-provider-reconciliation.test.ts b/test/onboarding/onboard-extra-provider-reconciliation.test.ts index 7791e2d8db0..256c7ad4b1a 100644 --- a/test/onboarding/onboard-extra-provider-reconciliation.test.ts +++ b/test/onboarding/onboard-extra-provider-reconciliation.test.ts @@ -92,11 +92,13 @@ runner.run = (command, opts = {}) => { if (normalized.includes("provider get -g nemoclaw ")) { return { status: 0, stdout: "" }; } - if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) { +if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) { if (sandboxCreated) { return { status: 0, - stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), + stdout: Buffer.from( + "my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n", + ), stderr: Buffer.alloc(0), }; } @@ -117,7 +119,9 @@ runner.runCapture = (command) => { : null; if (createdIdentity !== null) return createdIdentity; if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) { - return sandboxCreated ? "my-assistant\nId: sbx-4f2a91c0d7" : ""; + return sandboxCreated + ? "my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + : ""; } if (normalized.includes("sandbox list")) return sandboxCreated ? "my-assistant Ready" : ""; const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -192,7 +196,8 @@ const createReservedSandbox = () => createSandbox( const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", - timeout: 30_000, + timeout: 60_000, + killSignal: "SIGKILL", env: { ...process.env, HOME: tmpDir, diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index 05a3e23c2cf..76b29a34030 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -787,10 +787,21 @@ if (${JSON.stringify( assertSuccessfulCreation(); assert.equal(payload.registeredSandbox.policyAuthority, "externally-managed"); assert.equal(payload.registeredSandbox.policyCreationReceipt, undefined); - assert.doesNotMatch(payload.createCommand, /(?:^|\s)--policy(?:\s|$)/u); + assert.deepEqual(payload.registeredSandbox.appliedPolicies ?? [], []); + assert.doesNotMatch(payload.createCommand, /(?:^|\s)--policy(?:=|\s)/u); assert.doesNotMatch(payload.createCommand, /(?:^|\s)--provider(?:\s|$)/u); assert.equal(payload.credentialReadCalls, 0); assert.deepEqual(providerExposureCommands, []); + const createIndex = payload.commandNames.findIndex((command: string) => + command.includes("sandbox create"), + ); + const deferredEffectIndexes = payload.commandNames + .map((command: string, index: number) => ({ command, index })) + .filter(({ command }: { command: string }) => + /provider (?:profile import|create)|sandbox provider attach/u.test(command), + ) + .map(({ index }: { index: number }) => index); + assert.ok(deferredEffectIndexes.every((index: number) => index > createIndex)); }; const assertPostCreateAuthorityRefusal = () => { assert.equal(payload.sandboxName, null); diff --git a/test/onboarding/onboard-installer-restore-intent.test.ts b/test/onboarding/onboard-installer-restore-intent.test.ts index 1fb1cc2b8ce..3754d86bb20 100644 --- a/test/onboarding/onboard-installer-restore-intent.test.ts +++ b/test/onboarding/onboard-installer-restore-intent.test.ts @@ -14,6 +14,7 @@ const repoRoot = path.join(import.meta.dirname, "../.."); const onboardScriptMocksPath = JSON.stringify( path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), ); +const ONBOARD_SUBPROCESS_TIMEOUT_MS = 30_000; const createdTmpDirs: string[] = []; function makeTmpDir(prefix: string): string { @@ -78,7 +79,7 @@ runner.run = (command) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { @@ -89,7 +90,7 @@ runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); if (createdIdentity !== null) return createdIdentity; } - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } @@ -217,7 +218,8 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 cwd: repoRoot, encoding: "utf-8", env, - timeout: 30_000, + timeout: ONBOARD_SUBPROCESS_TIMEOUT_MS, + killSignal: "SIGKILL", }); assert.equal(result.status, 0, result.stderr || result.error?.message); @@ -374,6 +376,8 @@ const { createSandbox } = require(${onboardPath}); const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", + timeout: ONBOARD_SUBPROCESS_TIMEOUT_MS, + killSignal: "SIGKILL", env: { ...process.env, HOME: tmpDir, @@ -441,7 +445,7 @@ runner.runCapture = (command) => { const normalized = _n(command); if (normalized.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; if (normalized.includes("policy get") && normalized.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); - if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ["my-assistant", "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)); if (normalized.includes("sandbox list")) return "my-assistant NotReady"; // Keep dashboard allocation inside this restore-intent fixture; host port // occupancy is unrelated to the not-ready decision under test. @@ -490,6 +494,8 @@ const { createSandbox } = require(${onboardPath}); cwd: repoRoot, encoding: "utf-8", env, + timeout: ONBOARD_SUBPROCESS_TIMEOUT_MS, + killSignal: "SIGKILL", }); assert.notEqual( diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index 359ea3883ad..65f7869122f 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -18,6 +18,7 @@ import { parseMessagingFixturePayload, writeCustomMessagingDockerfile, } from "../helpers/messaging-plan-fixtures"; +import { runBoundedOnboardScript } from "../helpers/onboard-child-process-harness"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; type CommandEntry = { @@ -30,9 +31,7 @@ type CommandEntry = { providerRevisions?: Record | null; rawCredentialInEnv?: boolean; }; - const parseStdoutJson = parseMessagingFixturePayload; - const repoRoot = path.join(import.meta.dirname, "../.."); const requireForTest = createRequire(import.meta.url); const yamlModulePath = requireForTest.resolve("yaml"); @@ -159,9 +158,8 @@ const { createSandbox, setupMessagingChannels } = require(${onboardPath}); }); `; fs.writeFileSync(scriptPath, script); - const result = spawnSync(process.execPath, [scriptPath], { + const result = runBoundedOnboardScript(scriptPath, { cwd: repoRoot, - encoding: "utf-8", env: { ...process.env, HOME: tmpDir, @@ -510,7 +508,9 @@ const { createSandbox } = require(${onboardPath}); const preflightPath = JSON.stringify(path.join(repoRoot, "src/lib/onboard/preflight.ts")); const credentialsPath = JSON.stringify(path.join(repoRoot, "src/lib/credentials/store.ts")); const telegramCredentialKeys = [ - "TELEGRAM_BOT_TOKEN", "TELEGRAM_BOT_TOKEN_AGENT_A", "TELEGRAM_BOT_TOKEN_AGENT_B", + "TELEGRAM_BOT_TOKEN", + "TELEGRAM_BOT_TOKEN_AGENT_A", + "TELEGRAM_BOT_TOKEN_AGENT_B", ]; const providerCredentialKeys = { "compatible-endpoint": ["COMPATIBLE_API_KEY"], @@ -539,7 +539,7 @@ runner.run = (command, opts = {}) => { const refresh = normalized.match(/provider update -g nemoclaw ([^ ]+)$/)?.[1]; if (refresh && gatewaySecrets.has(refresh)) { if (refresh === process.env.NEMOCLAW_TEST_FAIL_PROVIDER) return { status: 1 }; revisions.set(refresh, revisions.get(refresh) + 1); return { status: 0 }; } if (normalized.includes("provider get")) return { status: 1 }; - return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; + return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); @@ -589,7 +589,10 @@ const { createSandbox } = require(${onboardPath}); NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_TEST_FAIL_PROVIDER: failedProvider || "", ...Object.fromEntries( - [...Object.values(providerCredentialKeys).flat(), "GITHUB_TOKEN"].map((key) => [key, ""]), + [...Object.values(providerCredentialKeys).flat(), "GITHUB_TOKEN"].map((key) => [ + key, + "", + ]), ), }, }); @@ -716,7 +719,7 @@ runner.run = (command, opts = {}) => { commands.push({ command: normalized, env: opts.env || null }); if (normalized.includes("provider get -g nemoclaw my-assistant-telegram-bridge")) return { status: 0, stdout: "Name: my-assistant-telegram-bridge\nType: nemoclaw-mcp-v1\nCredential keys: TELEGRAM_BOT_TOKEN\nConfig keys: \n" }; if (normalized.includes("provider get")) return { status: 1 }; - return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; + return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); @@ -786,11 +789,8 @@ const { createSandbox } = require(${onboardPath}); }); `; fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { + const result = runBoundedOnboardScript(scriptPath, { cwd: repoRoot, - encoding: "utf-8", - timeout: 30_000, env: { ...process.env, HOME: tmpDir, @@ -879,7 +879,7 @@ runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; if (normalized.includes("provider get")) return { status: 1 }; - return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; + return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); @@ -1049,7 +1049,7 @@ runner.run = (command, opts = {}) => { const normalized = _n(command); commands.push({ command: normalized, env: opts.env || null }); if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; if (normalized.includes("provider get")) return { status: 1 }; - return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; + return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); @@ -1283,7 +1283,7 @@ runner.run = require(${onboardScriptMocksPath}).createStatefulMessagingProviderR runner.runCapture = (command) => { // Existing sandbox that is ready if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) { - return "Name: my-assistant\nId: sbx-4f2a91c0d7\n"; + return "Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n"; } if (_n(command).includes("sandbox list")) return "my-assistant Ready"; // All messaging providers already exist in gateway @@ -1526,7 +1526,7 @@ const { EventEmitter } = require("node:events"); const commands = []; runner.run = (command, opts = {}) => { commands.push({ command: _n(command), env: opts.env || null }); if (_n(command).includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; - return _n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; + return _n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); diff --git a/test/onboarding/onboard-prepared-build-context.test.ts b/test/onboarding/onboard-prepared-build-context.test.ts index a9db367e272..a0a792deb63 100644 --- a/test/onboarding/onboard-prepared-build-context.test.ts +++ b/test/onboarding/onboard-prepared-build-context.test.ts @@ -156,7 +156,7 @@ runner.run = (command) => { const profileResult = require(${onboardScriptMocksPath}).mockManagedEndpointlessProviderProfileRun(command); if (profileResult !== null) return profileResult; return normalized.includes("sandbox get") && normalized.includes(sandboxName) - ? { status: 0, stdout: Buffer.from(sandboxName + "\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from(sandboxName + "\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runFile = (file, args = []) => { @@ -182,7 +182,9 @@ runner.runCapture = (command) => { ].join("\n"); } if (normalized.includes("sandbox get")) { - return sandboxCreated ? sandboxName + "\nId: sbx-4f2a91c0d7\n" : ""; + return sandboxCreated + ? sandboxName + "\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n" + : ""; } if (normalized.includes("sandbox list")) return sandboxCreated ? sandboxName + " Ready" : ""; return ""; diff --git a/test/onboarding/onboard-prepared-gateway-handoff.test.ts b/test/onboarding/onboard-prepared-gateway-handoff.test.ts index 43604305a35..3ce98b579bf 100644 --- a/test/onboarding/onboard-prepared-gateway-handoff.test.ts +++ b/test/onboarding/onboard-prepared-gateway-handoff.test.ts @@ -131,7 +131,7 @@ const { onboard } = require(${onboardPath}); const result = runOnboardProcess(["--require", sourceRequireHook, scriptPath], { env: minimalSpawnEnv(home), - timeoutMs: 15_000, + timeoutMs: 45_000, }); try { @@ -142,7 +142,7 @@ const { onboard } = require(${onboardPath}); } } -describe("prepared DCode gateway handoff", () => { +describe("prepared DCode gateway handoff", { timeout: 60_000 }, () => { it("preserves the recorded gateway into the initial onboard flow (#6195)", () => { assert.deepEqual(runHandoffScenario("prepared"), { error: null, diff --git a/test/onboarding/onboard-reservation-recreate.test.ts b/test/onboarding/onboard-reservation-recreate.test.ts index 58adb17c3d5..e0cd4764144 100644 --- a/test/onboarding/onboard-reservation-recreate.test.ts +++ b/test/onboarding/onboard-reservation-recreate.test.ts @@ -106,14 +106,14 @@ runner.run = (command) => { return { status: 0, stdout: "No sandboxes found.\n" }; } return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { const cmd = _n(command); const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); if (createdIdentity !== null) return createdIdentity; - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } @@ -177,7 +177,7 @@ const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, sessionId: "session-owner", getSandbox: registry.getSandbox, removeSandbox, - sourceSandboxId: "sbx-4f2a91c0d7", + sourceSandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID, }); const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"))}); diff --git a/test/onboarding/onboard-sandbox-build.test.ts b/test/onboarding/onboard-sandbox-build.test.ts index b78d0380f9d..7c0c269d95d 100644 --- a/test/onboarding/onboard-sandbox-build.test.ts +++ b/test/onboarding/onboard-sandbox-build.test.ts @@ -70,7 +70,7 @@ runner.run = (command, opts = {}) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return normalized.includes("sandbox get") && normalized.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { @@ -300,7 +300,7 @@ runner.run = (command, opts = {}) => { return sandboxCreated && normalized.includes("sandbox get") && normalized.split(/\s+/).includes("hermes-sandbox") - ? { status: 0, stdout: Buffer.from("Name: hermes-sandbox\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("Name: hermes-sandbox\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { @@ -316,7 +316,7 @@ runner.runCapture = (command) => { normalized.includes("sandbox get") && normalized.split(/\s+/).includes("hermes-sandbox") ) { - return "Name: hermes-sandbox\nId: sbx-4f2a91c0d7\nPhase: Ready\n"; + return "Name: hermes-sandbox\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"; } if (normalized.includes("sandbox list")) return "hermes-sandbox Ready"; { @@ -528,7 +528,7 @@ runner.run = (command, opts = {}) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return normalized.includes("sandbox get") && normalized.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { @@ -540,7 +540,9 @@ runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command, { sandboxName: "my-assistant" }); if (createdIdentity !== null) return createdIdentity; if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) { - return sandboxCreated ? "Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n" : ""; + return sandboxCreated + ? "Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n" + : ""; } if (normalized.includes("sandbox list")) return "my-assistant Ready"; { @@ -651,7 +653,7 @@ runner.run = (command, opts = {}) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return normalized.includes("sandbox get") && normalized.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => { @@ -765,7 +767,7 @@ runner.run = (command, opts = {}) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return normalized.includes("sandbox get") && normalized.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\nPhase: Ready\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\nPhase: Ready\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { diff --git a/test/onboarding/onboard-sandbox-recreation.test.ts b/test/onboarding/onboard-sandbox-recreation.test.ts index 738b37435b5..ec361f0a813 100644 --- a/test/onboarding/onboard-sandbox-recreation.test.ts +++ b/test/onboarding/onboard-sandbox-recreation.test.ts @@ -9,7 +9,7 @@ import os from "node:os"; import path from "node:path"; import { beforeEach, describe, it, vi } from "vitest"; -import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; +import { ONBOARD_CREATED_SANDBOX_ID, writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; import { type CommandEntry, onboardScriptMocksPath } from "../helpers/onboard-split-context"; beforeEach(() => { @@ -61,7 +61,7 @@ runner.run = (command) => { if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); // Existing sandbox that is NOT ready - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant NotReady"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; @@ -69,7 +69,7 @@ runner.run = (command) => { registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", toolDisclosure: "progressive", - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); childProcess.spawn = () => { throw new Error("unexpected sandbox create"); }; @@ -137,7 +137,7 @@ const { createSandbox } = require(${onboardPath}); const fixtureMocks = require(${onboardScriptMocksPath}); fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; +let _deleted = false; let _sandboxId = fixtureMocks.ONBOARD_CREATED_SANDBOX_ID; const registry = require(${registryPath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); @@ -154,7 +154,7 @@ const commands = []; let registeredSandbox = null; reference: "openshell/sandbox-from:source", shared: false, }, - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); runner.run = (command, opts = {}) => { const cmd = _n(command); const profileResult = require(${onboardScriptMocksPath}).mockEndpointlessProviderProfileRun(command, "nemoclaw-mcp-v1", false); @@ -268,7 +268,9 @@ const { createSandbox } = require(${onboardPath}); ), "must defer source image retirement until replacement registration is proven", ); - const sourceFingerprint = createHash("sha256").update("sbx-4f2a91c0d7").digest("hex"); + const sourceFingerprint = createHash("sha256") + .update(ONBOARD_CREATED_SANDBOX_ID) + .digest("hex"); const replacementFingerprint = createHash("sha256").update("sbx-8e6b10fd33").digest("hex"); assert.match(payload.registeredSandbox?.lifecycleGeneration ?? "", /^[0-9a-f-]{36}$/); assert.equal( @@ -309,7 +311,7 @@ const { createSandbox } = require(${onboardPath}); const fixtureMocks = require(${onboardScriptMocksPath}); fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; + let _deleted = false; let _sandboxId = fixtureMocks.ONBOARD_CREATED_SANDBOX_ID; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -349,7 +351,7 @@ runner.run = (command) => { const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", gpuEnabled: false, - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); registry.getSandbox = () => sourceSandbox; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", @@ -495,7 +497,7 @@ const { createSandbox } = require(${onboardPath}); const fixtureMocks = require(${onboardScriptMocksPath}); fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; + let _deleted = false; let _sandboxId = fixtureMocks.ONBOARD_CREATED_SANDBOX_ID; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -535,7 +537,7 @@ runner.run = (command) => { const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", gpuEnabled: false, - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); registry.getSandbox = () => sourceSandbox; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", @@ -649,7 +651,7 @@ const { createSandbox } = require(${onboardPath}); const fixtureMocks = require(${onboardScriptMocksPath}); fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; + let _deleted = false; let _sandboxId = fixtureMocks.ONBOARD_CREATED_SANDBOX_ID; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -694,7 +696,7 @@ runner.run = (command) => { const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", gpuEnabled: false, - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); registry.getSandbox = () => sourceSandbox; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", @@ -827,7 +829,7 @@ const { createSandbox } = require(${onboardPath}); const fixtureMocks = require(${onboardScriptMocksPath}); fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; + let _deleted = false; let _sandboxId = fixtureMocks.ONBOARD_CREATED_SANDBOX_ID; const registry = require(${registryPath}); const onboardSession = require(${sessionModulePath}); const childProcess = require("node:child_process"); @@ -873,7 +875,7 @@ runner.run = (command, opts = {}) => { gpuEnabled: false, policies: ["npm"], policyTier: "balanced", - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); registry.getSandbox = () => sourceSandbox; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", @@ -1011,7 +1013,7 @@ runner.run = (command, opts = {}) => { return { status: 0, stdout: Buffer.from("No sandboxes found.\n"), stderr: Buffer.alloc(0) }; } return cmd.includes("sandbox get") && cmd.includes("my-assistant") - ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + ? { status: 0, stdout: Buffer.from("my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runFile = (file, args = [], opts = {}) => { @@ -1022,7 +1024,7 @@ runner.runFile = (file, args = [], opts = {}) => { const cmd = _n(command); if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; if (cmd.includes("policy get") && cmd.includes("--output json")) return JSON.stringify({ scope: "sandbox", sandbox: "my-assistant", status: "effective", policy_source: "sandbox", hash: "fixture-policy", active_version: 1, policy: {} }); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; @@ -1030,7 +1032,7 @@ runner.runFile = (file, args = [], opts = {}) => { registry.getSandbox = () => fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", toolDisclosure: "progressive", - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); // Mock prompt to return "y" (reuse) credentials.prompt = async () => "y"; @@ -1132,7 +1134,7 @@ const { createSandbox } = require(${onboardPath}); const fixtureMocks = require(${onboardScriptMocksPath}); fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; + let _deleted = false; let _sandboxId = fixtureMocks.ONBOARD_CREATED_SANDBOX_ID; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1194,7 +1196,7 @@ runner.runFile = (file, args = [], opts = {}) => { const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", toolDisclosure: "progressive", - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); registry.getSandbox = () => sourceSandbox; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", @@ -1314,7 +1316,7 @@ const { createSandbox } = require(${onboardPath}); const fixtureMocks = require(${onboardScriptMocksPath}); fixtureMocks.mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - let _deleted = false; let _sandboxId = "sbx-4f2a91c0d7"; + let _deleted = false; let _sandboxId = fixtureMocks.ONBOARD_CREATED_SANDBOX_ID; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1360,7 +1362,7 @@ runner.run = (command, opts = {}) => { const sourceSandbox = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", toolDisclosure: "progressive", - }, { sandboxId: "sbx-4f2a91c0d7" }); + }, { sandboxId: fixtureMocks.ONBOARD_CREATED_SANDBOX_ID }); registry.getSandbox = () => sourceSandbox; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", diff --git a/test/onboarding/onboard-script-mocks-contract.test.ts b/test/onboarding/onboard-script-mocks-contract.test.ts new file mode 100644 index 00000000000..b8f8ab945a3 --- /dev/null +++ b/test/onboarding/onboard-script-mocks-contract.test.ts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { ONBOARD_CREATED_SANDBOX_ID, writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; + +type CommandResult = { + status: number; + stdout?: Buffer; + stderr?: Buffer; +}; + +type Runner = { + run: (command: readonly string[], options?: Record) => CommandResult; + runCapture: (command: readonly string[], options?: Record) => string; +}; + +type OnboardScriptMocks = { + ONBOARD_CREATED_SANDBOX_ID: string; + createStatefulMessagingProviderRunner: (options: { + commands: Array<{ command: string }>; + readySandboxName: string; + }) => (command: readonly string[]) => CommandResult; + managedSandboxPolicyReceiptFixture: ( + entry: { name: string }, + options?: { sandboxId?: string }, + ) => { lifecycleLiveIdentityFingerprint: string }; + mockCreatedSandboxIdentityList: ( + command: readonly string[], + options?: { sandboxName?: string; sandboxId?: string }, + ) => string | null; + mockDockerSandboxLifecycleReleaseFromRunner: () => void; +}; + +const requireForTest = createRequire(import.meta.url); +const fixtureMocks = requireForTest("../helpers/onboard-script-mocks.cjs") as OnboardScriptMocks; +const runner = requireForTest("../../src/lib/runner.ts") as Runner; +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("shared onboarding process fixture contracts", () => { + it("uses one durable sandbox ID across create discovery and structured OpenShell probes", () => { + const fakeRoot = mkdtempSync(join(tmpdir(), "nemoclaw-onboard-fixture-contract-")); + temporaryDirectories.push(fakeRoot); + const fakeBin = join(fakeRoot, "bin"); + mkdirSync(fakeBin); + writeOkOpenshell(fakeBin, { readySandboxGet: true }); + + const createAttemptNonce = "a".repeat(62); + const createAttemptCommand = [ + "openshell", + "sandbox", + "list", + "-g", + "nemoclaw", + "--selector", + `ai.nvidia.nemoclaw.create-attempt=${createAttemptNonce}`, + "--output", + "json", + "--limit", + "2", + ]; + const createAttemptList = fixtureMocks.mockCreatedSandboxIdentityList(createAttemptCommand); + const sandboxGet = spawnSync( + join(fakeBin, "openshell"), + ["sandbox", "get", "-g", "nemoclaw", "my-assistant"], + { + encoding: "utf8", + timeout: 5_000, + killSignal: "SIGKILL", + }, + ); + const receipt = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant" }); + const messagingRunner = fixtureMocks.createStatefulMessagingProviderRunner({ + commands: [], + readySandboxName: "my-assistant", + }); + const messagingGet = messagingRunner([ + "openshell", + "sandbox", + "get", + "-g", + "nemoclaw", + "my-assistant", + ]); + + expect(fixtureMocks.ONBOARD_CREATED_SANDBOX_ID).toBe(ONBOARD_CREATED_SANDBOX_ID); + expect(JSON.parse(createAttemptList ?? "[]")).toEqual([ + expect.objectContaining({ + id: ONBOARD_CREATED_SANDBOX_ID, + labels: { "ai.nvidia.nemoclaw.create-attempt": createAttemptNonce }, + name: "my-assistant", + }), + ]); + expect( + fixtureMocks.mockCreatedSandboxIdentityList( + createAttemptCommand.map((argument) => + argument.replace( + "ai.nvidia.nemoclaw.create-attempt=", + "aiXnvidiaXnemoclawXcreate-attempt=", + ), + ), + ), + "the selector label prefix must match literally", + ).toBeNull(); + expect(sandboxGet.status, sandboxGet.stderr).toBe(0); + expect(sandboxGet.stdout).toContain(`Id: ${ONBOARD_CREATED_SANDBOX_ID}`); + expect(String(messagingGet.stdout)).toContain(`Id: ${ONBOARD_CREATED_SANDBOX_ID}`); + expect(receipt.lifecycleLiveIdentityFingerprint).toBe( + createHash("sha256").update(ONBOARD_CREATED_SANDBOX_ID).digest("hex"), + ); + }); + + it("composes Docker lifecycle state across run and runCapture", () => { + const originalRun = runner.run; + const originalRunCapture = runner.runCapture; + const readyList = "my-assistant 2026-08-27 Ready\n"; + const listCommand = ["openshell", "sandbox", "list"]; + runner.run = () => ({ + status: 0, + stdout: Buffer.from(readyList), + stderr: Buffer.alloc(0), + }); + runner.runCapture = () => readyList; + + try { + fixtureMocks.mockDockerSandboxLifecycleReleaseFromRunner(); + const oldContainerId = "a".repeat(64); + const newContainerId = "b".repeat(64); + const containerListCommand = [ + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "label=openshell.ai/sandbox-name=my-assistant", + "--format", + "{{.ID}}", + ]; + + expect(runner.runCapture(listCommand)).toBe(readyList); + expect(runner.run(["docker", "rm", oldContainerId]).status).toBe(0); + expect(String(runner.run(containerListCommand).stdout)).toBe(`${newContainerId}\n`); + expect(runner.runCapture(containerListCommand)).toBe(`${newContainerId}\n`); + + expect(runner.run(["openshell", "sandbox", "stop", "my-assistant"]).status).toBe(0); + expect(String(runner.run(listCommand).stdout)).toContain("Stopped"); + expect(runner.runCapture(listCommand)).toContain("Stopped"); + + expect(runner.run(["openshell", "sandbox", "start", "my-assistant"]).status).toBe(0); + expect(String(runner.run(listCommand).stdout)).toContain("Ready"); + expect(runner.runCapture(listCommand)).toContain("Ready"); + } finally { + runner.run = originalRun; + runner.runCapture = originalRunCapture; + } + }); +}); diff --git a/test/onboarding/onboard-terminal-dashboard.test.ts b/test/onboarding/onboard-terminal-dashboard.test.ts index 546ffec1342..08f5a3f3ce1 100644 --- a/test/onboarding/onboard-terminal-dashboard.test.ts +++ b/test/onboarding/onboard-terminal-dashboard.test.ts @@ -127,7 +127,13 @@ runner.run = (command, opts = {}) => { const providerResult = managedProviderResult(normalized); return profileResult ?? providerResult ?? (normalized.includes("sandbox get") && normalized.includes(sandboxName) - ? { status: 0, stdout: Buffer.from("Name: " + sandboxName + "\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } + ? { + status: 0, + stdout: Buffer.from( + "Name: " + sandboxName + "\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n", + ), + stderr: Buffer.alloc(0), + } : { status: 0 }); }; runner.runFile = (file, args = [], opts = {}) => { @@ -157,7 +163,7 @@ runner.runCapture = (command) => { } if (normalized.includes("sandbox get") && normalized.includes(sandboxName)) { return scenario === "reuse" - ? [sandboxName, "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) + ? [sandboxName, "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)) : ""; } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; @@ -249,7 +255,8 @@ const agent = agentDefs.loadAgent("langchain-deepagents-code"); NEMOCLAW_TEST_MANAGED_IMAGE_CATALOG: "1", OPENSHELL_DRIVERS: scenario === "create" ? "vm" : "docker", }, - timeout: 15000, + timeout: 30_000, + killSignal: "SIGKILL", }); assert.equal(result.status, 0, result.stderr); return parseStdoutJson<{ diff --git a/test/onboarding/onboard.test.ts b/test/onboarding/onboard.test.ts index ce0a00afc49..fba42ff0422 100644 --- a/test/onboarding/onboard.test.ts +++ b/test/onboarding/onboard.test.ts @@ -696,7 +696,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; diff --git a/test/security/shellquote-sandbox.test.ts b/test/security/shellquote-sandbox.test.ts index 4bde66e0631..767cb92c897 100644 --- a/test/security/shellquote-sandbox.test.ts +++ b/test/security/shellquote-sandbox.test.ts @@ -108,7 +108,9 @@ runner.run = (command, opts = {}) => { if (text.includes("sandbox get") && text.includes("my-assistant")) { return { status: 0, - stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), + stdout: Buffer.from( + "Name: my-assistant\nId: " + fixtureMocks.ONBOARD_CREATED_SANDBOX_ID + "\n", + ), stderr: Buffer.alloc(0), }; }