Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"src/lib/adapters/openshell/client.ts": 19,
"src/lib/adapters/openshell/command-argv.ts": 22,
"src/lib/adapters/openshell/resolve.ts": 25,
"src/lib/adapters/openshell/runtime.ts": 47,
"src/lib/adapters/openshell/runtime.ts": 46,
"src/lib/adapters/openshell/sandbox-observer.ts": 35,
"src/lib/adapters/openshell/timeouts.ts": 32,
"src/lib/agent/defs.ts": 32,
Expand All @@ -24,7 +24,7 @@
"src/lib/core/wait.ts": 30,
"src/lib/credentials/store.ts": 45,
"src/lib/inference/config.ts": 30,
"src/lib/messaging/channels/index.ts": 25,
"src/lib/messaging/channels/index.ts": 24,
"src/lib/onboard/gateway-binding.ts": 53,
"src/lib/runner.ts": 83,
"src/lib/security/redact.ts": 52,
Expand Down
1 change: 1 addition & 0 deletions src/commands/credentials.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ describe("credentials oclif adapter source coverage", () => {
{
ignoreError: true,
stdio: ["ignore", "pipe", "pipe"],
suppressOutput: true,
timeout: 30_000,
},
);
Expand Down
67 changes: 67 additions & 0 deletions src/commands/internal/uninstall/run-plan.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";
import type { Config } from "@oclif/core";

const mocks = vi.hoisted(() => ({
runUninstallPlan: vi.fn(),
runUninstallPlanProduction: vi.fn(),
}));

vi.mock("../../../lib/actions/uninstall/run-plan", () => mocks);
vi.mock("../../../lib/actions/maintenance", () => ({ backupAllUnderPortableHostFence: vi.fn() }));
vi.mock("../../../lib/actions/uninstall/all-gateway-ports", () => ({
allGatewayPortsRequested: () => false,
runUninstallAllGatewayPorts: vi.fn(),
}));

import InternalUninstallRunPlanCommand from "./run-plan";

describe("internal uninstall command", () => {
it("awaits the production uninstall transaction before applying its exit result", async () => {
const previousExitCode = process.exitCode;
let complete!: (value: { exitCode: number }) => void;
let entered!: () => void;
const started = new Promise<void>((resolve) => {
entered = resolve;
});
const pending = new Promise<{ exitCode: number }>((resolve) => {
complete = resolve;
});
mocks.runUninstallPlanProduction.mockImplementation(() => {
entered();
return pending;
});
let finished = false;
process.exitCode = undefined;
const instance = new InternalUninstallRunPlanCommand([], {} as Config);
Object.defineProperty(instance, "parse", {
value: vi.fn().mockResolvedValue({ flags: { yes: true, "keep-openshell": true } }),
});
const command = instance.run().then(() => {
finished = true;
});
try {
await Promise.race([started, command]);
expect(mocks.runUninstallPlanProduction).toHaveBeenCalledExactlyOnceWith(
expect.objectContaining({ assumeYes: true, keepOpenShell: true }),
expect.objectContaining({
backupAllBeforeUninstall: expect.any(Function),
withSandboxMutationLock: expect.any(Function),
}),
);
expect(mocks.runUninstallPlan).not.toHaveBeenCalled();
expect(finished).toBe(false);
expect(process.exitCode).toBeUndefined();
complete({ exitCode: 1 });
await command;
expect(process.exitCode).toBe(1);
} finally {
complete({ exitCode: 0 });
await command.finally(() => {
process.exitCode = previousExitCode;
});
}
});
});
9 changes: 6 additions & 3 deletions src/lib/actions/sandbox/destroy-execution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,7 @@ export async function executeSandboxDestroy({
mcpRecoveryFailure,
};
}
const detachProviders = (): DetachSandboxProvidersResult =>
const detachProviders = (): Promise<DetachSandboxProvidersResult> =>
runSandboxProviderPreDeleteCleanup(sandboxName, {
runOpenshell: selectedRunOpenshell,
redact,
Expand All @@ -545,8 +545,11 @@ export async function executeSandboxDestroy({
const detachOutcome: DetachSandboxProvidersResult = sandboxConfirmedAbsent
? { detached: [], failures: [] }
: runtimeProvider?.cleanup.supported === true && sandbox
? runtimeProvider.cleanup.prepareDestroy({ sandbox, sandboxName }, { detachProviders })
: detachProviders();
? await runtimeProvider.cleanup.prepareDestroy(
{ sandbox, sandboxName },
{ detachProviders },
)
: await detachProviders();
// The final identity proof runs immediately before OpenShell delete. A
// runtime administrator remains a trusted host authority; this closes the
// multi-step window without claiming a cross-runtime transaction.
Expand Down
32 changes: 32 additions & 0 deletions src/lib/actions/sandbox/destroy-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,38 @@ describe("destroySandbox flow", () => {
expectStrictSandboxPresenceClassification();
});

it("waits for provider detach before deleting the sandbox", { timeout: 30_000 }, async () => {
const harness = createDestroyHarness();
let finishDetach!: () => void;
let detachStarted!: () => void;
const pending = new Promise<void>((resolve) => {
finishDetach = resolve;
});
const started = new Promise<void>((resolve) => {
detachStarted = resolve;
});
harness.runSandboxProviderPreDeleteCleanupSpy.mockImplementationOnce(async () => {
detachStarted();
await pending;
return { detached: [], failures: [] };
});
const destroy = harness.destroySandbox("alpha", { yes: true, cleanupGateway: false });
try {
await Promise.race([started, destroy]);
expect(harness.runSandboxProviderPreDeleteCleanupSpy).toHaveBeenCalledOnce();
expect(harness.events).not.toContain("delete");
expect(harness.removeSandboxSpy).not.toHaveBeenCalled();
expect(harness.lifecycleLockEvents).toContain("acquired");
expect(harness.lifecycleLockEvents).not.toContain("released");
} finally {
finishDetach();
await destroy;
}
expect(harness.events).toContain("delete");
expect(harness.removeSandboxSpy).toHaveBeenCalledOnce();
expect(harness.lifecycleLockEvents).toContain("released");
});

it(
"selects the sandbox gateway, deletes live resources, cleans host state, and removes registry state",
{ timeout: 30_000 },
Expand Down
46 changes: 39 additions & 7 deletions src/lib/actions/sandbox/destroy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,46 @@ import path from "node:path";
import { describe, expect, it, vi } from "vitest";

import type { SandboxEntry } from "../../state/registry";
import { SANDBOX_PROVIDER_SUFFIXES } from "../../onboard/sandbox-provider-cleanup";
import { assertUnambiguousDestroyContainerIdentity, cleanupSandboxServices } from "./destroy";

const SANDBOX = "mybox";
const mainPidDir = path.resolve("/tmp", `nemoclaw-services-${SANDBOX}`);
const googlechatPidDir = `${mainPidDir}-googlechat`;

describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => {
it("fails closed before later cleanup when the Google Chat tunnel cannot stop", () => {
it("deletes every messaging and search registration through the selected gateway", async () => {
const runOpenshell = vi.fn(() => ({ status: 0 }));

await cleanupSandboxServices(
SANDBOX,
{ stopHostServices: false },
{
getSandbox: () => null,
listSandboxes: () => ({ sandboxes: [], defaultSandbox: null }),
withOllamaModelOwnershipLock: (operation) => operation(),
loadPersistedOllamaHost: () => "127.0.0.1",
loadPendingOllamaModelCleanup: () => [],
rmSync: vi.fn(),
runOpenshell,
stopGooglechatWebhookTunnel: vi.fn(() => googlechatPidDir),
googlechatWebhookTunnelPidDir: vi.fn(() => googlechatPidDir),
},
);

expect(runOpenshell.mock.calls).toEqual(
SANDBOX_PROVIDER_SUFFIXES.map((suffix) => [
["provider", "delete", `${SANDBOX}-${suffix}`],
expect.objectContaining({
ignoreError: true,
suppressOutput: true,
stdio: ["ignore", "pipe", "pipe"],
}),
]),
);
});

it("fails closed before later cleanup when the Google Chat tunnel cannot stop", async () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
const rmSync = vi.fn();
const runOpenshell = vi.fn(() => ({ status: 0 }));
Expand All @@ -23,7 +55,7 @@ describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => {
throw new Error("cloudflared refused to stop");
});

expect(() =>
await expect(
cleanupSandboxServices(
SANDBOX,
{ stopHostServices: true },
Expand All @@ -36,7 +68,7 @@ describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => {
googlechatWebhookTunnelPidDir,
},
),
).toThrow(/public Google Chat webhook endpoint may still be running/);
).rejects.toThrow(/public Google Chat webhook endpoint may still be running/);

expect(googlechatWebhookTunnelPidDir).toHaveBeenCalledWith(mainPidDir);
// Preserve both PID directories and refuse every later side effect so a
Expand All @@ -50,12 +82,12 @@ describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => {
warn.mockRestore();
});

it("removes the Google Chat PID directory after a successful tunnel stop", () => {
it("removes the Google Chat PID directory after a successful tunnel stop", async () => {
const rmSync = vi.fn();
const stopGooglechatWebhookTunnel = vi.fn(() => googlechatPidDir);
const googlechatWebhookTunnelPidDir = vi.fn(() => googlechatPidDir);

cleanupSandboxServices(
await cleanupSandboxServices(
SANDBOX,
{ stopHostServices: true },
{
Expand All @@ -73,7 +105,7 @@ describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => {
});

describe("cleanupSandboxServices Ollama ownership", () => {
it("keeps a model shared through a compatible endpoint at the same local daemon", () => {
it("keeps a model shared through a compatible endpoint at the same local daemon", async () => {
const own = {
name: SANDBOX,
provider: "ollama-local",
Expand All @@ -87,7 +119,7 @@ describe("cleanupSandboxServices Ollama ownership", () => {
} as SandboxEntry;
const unloadOllamaModels = vi.fn();

cleanupSandboxServices(
await cleanupSandboxServices(
SANDBOX,
{ stopHostServices: false },
{
Expand Down
22 changes: 10 additions & 12 deletions src/lib/actions/sandbox/destroy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ import {
} from "../../onboard/runtime-provider/access";
import {
emitProviderDetachResidualHint,
deleteSandboxProviderRegistrations,
removeManagedAgentStateVolumes,
SANDBOX_PROVIDER_SUFFIXES,
} from "../../onboard/sandbox-provider-cleanup";
import { validateName } from "../../runner";
import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock";
Expand Down Expand Up @@ -222,11 +222,11 @@ async function resolveCleanupGatewayDecision(options: DestroySandboxOptions): Pr
return trimmed === "y" || trimmed === "yes";
}

export function cleanupSandboxServices(
export async function cleanupSandboxServices(
sandboxName: string,
{ stopHostServices = false }: { stopHostServices?: boolean } = {},
deps: CleanupSandboxServicesDeps = {},
): void {
): Promise<void> {
// Source boundary: this exported helper can be called independently of CLI
// dispatch, including from forced local recovery. Validate once before every
// host and provider cleanup side effect, then derive the PID path from that
Expand Down Expand Up @@ -442,12 +442,7 @@ export function cleanupSandboxServices(
// onboard rebuild path's pre-delete detach via
// `src/lib/onboard/sandbox-provider-cleanup.ts` so the two paths can't
// drift on which providers count as per-sandbox state.
for (const suffix of SANDBOX_PROVIDER_SUFFIXES) {
runOpenshell(["provider", "delete", `${validatedSandboxName}-${suffix}`], {
ignoreError: true,
stdio: ["ignore", "ignore", "ignore"],
});
}
await deleteSandboxProviderRegistrations(validatedSandboxName, "all", { runOpenshell });
}

/**
Expand Down Expand Up @@ -1002,13 +997,13 @@ async function destroySandboxUnlocked(
}
}
}
abortPreparedCleanupOnError(() => {
try {
const shouldStopHostServices = shouldStopHostServicesAfterDestroy({
deleteSucceededOrAlreadyGone,
registeredSandboxCount: registry.listSandboxes().sandboxes.length,
sandboxStillRegistered: !!registry.getSandbox(sandboxName),
});
cleanupSandboxServices(
await cleanupSandboxServices(
sandboxName,
{
stopHostServices: shouldStopHostServices,
Expand All @@ -1017,7 +1012,10 @@ async function destroySandboxUnlocked(
runOpenshell: cleanupRunOpenshell,
},
);
});
} catch (error) {
preparedManagedLlamaCppCleanup?.abort();
throw error;
}
if (deleteSucceededOrAlreadyGone && commonLlamaCppAuthorityRetired === true) {
preparedManagedLlamaCppCleanup?.abort();
} else if (deleteSucceededOrAlreadyGone) {
Expand Down
8 changes: 7 additions & 1 deletion src/lib/actions/sandbox/snapshot-auto-create-failure.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { beforeEach, describe, expect, it, vi } from "vitest";
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import {
hostLocalInferenceReceipt,
serializedLlamaCppHostLocalInferenceReceipt,
Expand Down Expand Up @@ -186,6 +186,7 @@ vi.mock("../../inference/nim", () => ({
}));
vi.mock("../../messaging/channels", () => ({
BUILT_IN_CHANNEL_MANIFESTS: [],
createBuiltInChannelManifestRegistry: vi.fn(() => ({ list: () => [] })),
getMessagingConfigEnvAliases: vi.fn(() => ({})),
getMessagingCredentialEnvKeysByChannel: vi.fn(() => ({})),
getMessagingProviderSuffixesByChannel: vi.fn(() => ({})),
Expand Down Expand Up @@ -268,6 +269,11 @@ vi.mock("./snapshot/dependencies", async (importOriginal) => ({
}));

describe("snapshot restore auto-create failures", () => {
beforeAll(async () => {
// Load the mocked action graph before measuring the individual cleanup operations.
await import("./snapshot");
}, 30_000);

beforeEach(() => {
vi.clearAllMocks();
harness.entries.clear();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import { createHash } from "node:crypto";

import { describe, expect, it, type Mock, vi } from "vitest";
import { createManagedProviderAdapter } from "../../adapters/openshell/managed-provider-adapter";
import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts";
import type { HermesToolGatewayCloneBroker } from "../../hermes-tool-gateway-clone-broker";
import {
Expand Down Expand Up @@ -413,6 +414,8 @@ describe("Hermes managed clone broker transaction", () => {
const profile = hermesProfile();
const source = sourceEntry(profile);
const runner = providerRunner();
const providerAdapter = createManagedProviderAdapter(runner.run);
const deleteProvider = vi.spyOn(providerAdapter, "deleteProvider");
const hostBroker = broker();
hostBroker.activateHermesToolGatewayCloneBinding.mockImplementation(() => {
throw new Error("activation rejected");
Expand All @@ -431,10 +434,12 @@ describe("Hermes managed clone broker transaction", () => {
...authority(source),
environment: environment(),
runOpenshell: runner.run,
providerAdapter,
broker: hostBroker,
}),
).rejects.toThrow("activation rejected");
expect(runner.live.size).toBe(0);
expect(deleteProvider).toHaveBeenCalledTimes(2);
expect(hostBroker.discardHermesToolGatewayCloneBinding).toHaveBeenCalledOnce();
});
});
Loading
Loading