Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
33c3bd2
fix(destroy): release final gateway on macOS
chengjiew Jul 9, 2026
cb23052
test(destroy): avoid platform assertion branches
chengjiew Jul 9, 2026
cf54c98
test(destroy): cover unattended macOS triggers
chengjiew Jul 9, 2026
66ddd11
test(destroy): cover Linux non-interactive default
chengjiew Jul 9, 2026
d4e2608
test(destroy): cover declined gateway cleanup
apurvvkumaria Jul 9, 2026
36ea126
test(destroy): cover native Windows gateway preservation
apurvvkumaria Jul 9, 2026
55e2178
test(destroy): cover gateway cleanup precedence
apurvvkumaria Jul 9, 2026
b799c8a
fix(destroy): document macOS gateway cleanup removal
prekshivyas Jul 10, 2026
bf0fd92
Merge remote-tracking branch 'origin/main' into fix/4662_release_maco…
prekshivyas Jul 10, 2026
a363d67
fix(onboard): keep noninteractive helper line neutral
prekshivyas Jul 10, 2026
8ff6ce2
Merge branch 'main' into fix/4662_release_macos_gateway_current
prekshivyas Jul 10, 2026
985968e
fix(onboard): keep noninteractive helper hoisted
prekshivyas Jul 10, 2026
746046c
fix(destroy): release macos gateway after final sandbox
prekshivyas Jul 10, 2026
d5b2a7f
test(onboard): keep gateway process test branchless
prekshivyas Jul 10, 2026
d976458
refactor(destroy): extract gateway cleanup probes
prekshivyas Jul 10, 2026
961a6af
test(destroy): wire macos final gateway cleanup coverage
prekshivyas Jul 10, 2026
a254dbe
test(e2e): record sandbox operations live parity
prekshivyas Jul 10, 2026
2e7b505
Merge remote-tracking branch 'upstream/main' into HEAD
prekshivyas Jul 10, 2026
3ae27f5
fix(ci): gate macos live e2e to trusted runs
prekshivyas Jul 10, 2026
d63f124
fix(e2e): make final sandbox cleanup explicit
prekshivyas Jul 10, 2026
57bfbf1
test(e2e): avoid growing sandbox conditional count
prekshivyas Jul 10, 2026
b9ca8ab
fix(destroy): isolate final gateway cleanup probe
prekshivyas Jul 10, 2026
714810a
fix(destroy): keep live probes out of domain
prekshivyas Jul 10, 2026
312da1f
test(e2e): prove macos default gateway cleanup
prekshivyas Jul 10, 2026
31592c9
Merge remote-tracking branch 'upstream/main' into HEAD
prekshivyas Jul 10, 2026
81d4a69
fix(destroy): scope host gateway pid cleanup
prekshivyas Jul 10, 2026
63da39c
test(destroy): keep gateway process mocks linear
prekshivyas Jul 10, 2026
7f1ef5f
fix(destroy): fail closed on gateway docker probe errors
prekshivyas Jul 10, 2026
b523514
fix(destroy): scope direct gateway pid cleanup
prekshivyas Jul 10, 2026
fe44e16
Merge remote-tracking branch 'upstream/main' into fix/4662_release_ma…
prekshivyas Jul 10, 2026
b97f572
fix(destroy): verify compat gateway cleanup target
prekshivyas Jul 10, 2026
890c63b
chore(onboard): keep entrypoint net-neutral
prekshivyas Jul 10, 2026
3a0293f
fix(destroy): bind host gateway launch identity
prekshivyas Jul 10, 2026
dce281d
Merge branch 'main' into fix/4662_release_macos_gateway_current
cjagwani Jul 10, 2026
f854fe4
fix(onboard): migrate legacy gateway identity
prekshivyas Jul 10, 2026
afdcd13
test(destroy): lock fail-closed cleanup paths
prekshivyas Jul 10, 2026
c38fff0
test(gateway): keep runtime regressions linear
prekshivyas Jul 10, 2026
15e27f2
Merge remote-tracking branch 'upstream/main' into fix/4662_release_ma…
prekshivyas Jul 10, 2026
1958a26
refactor(gateway): isolate upgrade identity checks
prekshivyas Jul 10, 2026
6489653
fix(gateway): close final destroy review gaps
prekshivyas Jul 10, 2026
90d4178
Merge remote-tracking branch 'upstream/main' into fix/4662_release_ma…
prekshivyas Jul 10, 2026
95cdb0b
test(gateway): keep retry scenarios branchless
prekshivyas Jul 10, 2026
4517772
fix(onboard): preserve authoritative rebuild provider
prekshivyas Jul 10, 2026
2fb6203
Merge remote-tracking branch 'upstream/main' into fix/4662_release_ma…
prekshivyas Jul 10, 2026
7233211
fix(gateway): fail final destroy on reaper errors
prekshivyas Jul 10, 2026
f4f7663
Merge branch 'main' into fix/4662_release_macos_gateway_current
cv Jul 11, 2026
337ab2d
fix(onboard): restore scoped provider recovery
cv Jul 11, 2026
ce28822
fix(destroy): reject unverifiable gateway pid
cv Jul 11, 2026
62c5037
Merge branch 'main' into fix/4662_release_macos_gateway_current
cv Jul 11, 2026
8ba894e
fix(destroy): preserve gateway runtime evidence
cv Jul 11, 2026
3174d61
test(destroy): keep gateway retry fixture linear
cv Jul 11, 2026
3162785
fix(destroy): honor non-interactive confirmation
cv Jul 11, 2026
3d4e84b
fix(destroy): surface Docker cleanup probe failures
cv Jul 11, 2026
d8a4437
docs(destroy): document unattended cleanup failures
cv Jul 11, 2026
934c506
fix(destroy): reject untargeted gateway process identity
cv Jul 11, 2026
f88f8c0
ci(macos): preserve final-destroy diagnostics
cv Jul 11, 2026
57a7ffb
Merge remote-tracking branch 'origin/main' into codex/fix-6597-exact-…
cv Jul 11, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 89 additions & 9 deletions .github/workflows/macos-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -75,23 +75,25 @@ jobs:
npx vitest run --project integration
test/tunnel-gateway-port-release-runtime.test.ts
test/onboard-gateway-prelaunch-cutover.test.ts
test/onboard-gateway-legacy-identity-upgrade-runtime.test.ts

- name: Detect Docker availability
id: docker
run: |
set -euo pipefail
if docker info >/dev/null 2>&1; then
echo "docker_ok=true" >> "$GITHUB_OUTPUT"
echo "Docker is available"
docker version
else
echo "docker_ok=false" >> "$GITHUB_OUTPUT"
echo "Docker is not available on this runner"
exit 0
fi
echo "docker_ok=false" >> "$GITHUB_OUTPUT"
echo "Docker is unavailable on the Apple Silicon runner."

- name: Run macOS full E2E
if: steps.docker.outputs.docker_ok == 'true'
if: steps.docker.outputs.docker_ok == 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
NVIDIA_INFERENCE_API_KEY: ${{ github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
GITHUB_TOKEN: ${{ github.token }}
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
Expand All @@ -100,17 +102,95 @@ jobs:
run: |
NEMOCLAW_RUN_LIVE_E2E=1 npx vitest run --project e2e-live test/e2e/live/full-e2e.test.ts --silent=false --reporter=default

- name: Explain skipped full E2E
if: steps.docker.outputs.docker_ok != 'true'
- name: Explain skipped macOS live E2E
if: steps.docker.outputs.docker_ok != 'true' || github.ref != 'refs/heads/main' || github.event_name == 'pull_request'
run: |
echo 'Skipping macOS full E2E because Docker is unavailable on this runner.'
if [ "${{ github.event_name }}" = "pull_request" ]; then
echo 'Skipping secret-bearing macOS live E2E on pull_request; use trusted workflow_dispatch/push evidence for live validation.'
elif [ "${{ github.ref }}" != "refs/heads/main" ]; then
echo 'Skipping secret-bearing macOS live E2E outside the trusted main branch.'
elif [ "${{ steps.docker.outputs.docker_ok }}" != "true" ]; then
echo 'Skipping macOS live E2E because Docker is unavailable on this runner.'
fi
echo 'The workflow still validated the NemoClaw build on macOS (Apple Silicon).'

- name: Upload logs on failure
if: failure()
if: failure() && github.event_name == 'pull_request'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: macos-e2e-logs
path: |
/tmp/nemoclaw-e2e-*.log
${{ github.workspace }}/e2e-artifacts/live
if-no-files-found: ignore

# docker/setup-docker-action supports Intel macOS, while the primary Apple
# Silicon job validates the same gateway lifecycle regressions without Docker.
# Keep the secret-bearing real Docker proof on reviewed main-branch code only.
macos-docker-final-destroy:
if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
permissions:
contents: read
runs-on: macos-15-intel
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: "22"
cache: npm

- name: Set up pinned Docker Engine
uses: docker/setup-docker-action@6d7cfa65f60a9dda7b46e5513fa982536f3c9877 # v5.3.0
with:
version: v27.4.0
env:
LIMA_START_ARGS: --cpus 4 --memory 8

- name: Install root dependencies
run: npm ci --ignore-scripts

- name: Build CLI TypeScript modules
run: npm run build:cli

- name: Install and build plugin
run: |
set -euo pipefail
cd nemoclaw
npm ci --ignore-scripts
npm run build

- name: Install OpenShell CLI
run: bash scripts/install-openshell.sh

- name: Run macOS Docker final-destroy E2E
env:
E2E_TARGET_ID: "sandbox-operations"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/macos-sandbox-operations
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_POLICY_TIER: "open"
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
OPENSHELL_GATEWAY: "nemoclaw"
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
npx vitest run --project e2e-live \
test/e2e/live/sandbox-operations.test.ts \
--silent=false --reporter=default

- name: Upload macOS Docker logs on failure
if: failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: macos-docker-final-destroy-logs
path: |
/tmp/nemoclaw-e2e-*.log
${{ github.workspace }}/e2e-artifacts/live
if-no-files-found: ignore
14 changes: 9 additions & 5 deletions docs/reference/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1506,7 +1506,7 @@ If you want to upgrade the sandbox while preserving state, use `$$nemoclaw <name
</Warning>

If another terminal has an active SSH session to the sandbox, `destroy` prints an active-session warning and requires a second confirmation before it proceeds.
Pass `--yes`, `-y`, or `--force` to skip the prompt in scripted workflows.
Pass `--yes`, `-y`, or `--force`, or set `NEMOCLAW_NON_INTERACTIVE=1`, to authorize deletion without prompting in scripted workflows.

<AgentOnly variant="hermes">

Expand All @@ -1521,10 +1521,14 @@ It restores and verifies lockdown and revokes the active timer before deletion.
It clears the remaining local shields state only after deletion succeeds.
If hardening fails, the command refuses deletion and leaves the timer authority available to retry lockdown.
If deletion fails after hardening, the command keeps the surviving sandbox's locked shields state instead of cleaning it up as though deletion succeeded.
By default, `destroy` preserves the shared NemoClaw gateway.
Pass `--cleanup-gateway` to remove the shared gateway when destroying the last sandbox, or `--no-cleanup-gateway` to force preservation when environment defaults request cleanup.
By default, unattended final-sandbox destroys (`--yes`, `--force`, or `NEMOCLAW_NON_INTERACTIVE=1`) remove the shared NemoClaw gateway on macOS so the host listener is released, while Linux preserves it for reuse.
Pass `--cleanup-gateway` to force removal, or `--no-cleanup-gateway` to force preservation.
These flags always override both `NEMOCLAW_CLEANUP_GATEWAY` and the platform default.
If the pre-delete workspace wipe cannot run, use a different sandbox name for a clean start.
When this is the last sandbox, pass `--cleanup-gateway` to purge the shared cluster volume that retains the per-name persistent volume.
Cleaning up the gateway after the last sandbox also purges the shared cluster volume that retains the per-name persistent volume.
If final gateway cleanup finds a live PID-file process whose command line does not prove it owns the target gateway, `destroy` exits non-zero after sandbox and registry deletion and skips gateway and volume removal.
NemoClaw preserves the per-gateway PID file and runtime marker so you can inspect the process.
Stop only the listener that matches the target gateway, then rerun `destroy` to converge cleanup.
If the OpenShell gateway is unreachable and the sandbox has no managed MCP ownership state, `--force` removes only NemoClaw's local registry entry and local artifacts.
Gateway-side deletion remains unconfirmed, shared host-service and gateway teardown are skipped, and the sandbox and retained volume may still exist if the gateway returns.
Start the gateway with `$$nemoclaw <name> status` and retry destroy when you need a confirmed deletion.
Expand Down Expand Up @@ -3501,7 +3505,7 @@ The following flags change defaults for commands that manage existing sandboxes.

| Variable | Format | Effect |
|----------|--------|--------|
| `NEMOCLAW_CLEANUP_GATEWAY` | `1`, `true`, or `yes` to enable; `0`, `false`, or `no` to disable | Sets the default for whether `$$nemoclaw <name> destroy` removes the shared gateway when destroying the last sandbox. Command-line `--cleanup-gateway` and `--no-cleanup-gateway` still take precedence. |
| `NEMOCLAW_CLEANUP_GATEWAY` | `1`, `true`, or `yes` to enable; `0`, `false`, or `no` to disable | Overrides the platform default (macOS unattended: cleanup; Linux/Windows: preserve) for whether `$$nemoclaw <name> destroy` removes the shared gateway when destroying the last sandbox. Command-line `--cleanup-gateway` and `--no-cleanup-gateway` still take precedence. |
| `NEMOCLAW_CONFIRM_LEGACY_MANAGED_RECREATE` | Exact JSON array of sandbox names | Confirms to the installer that the exact listed set of pre-fingerprint OpenClaw or Hermes sandboxes used NemoClaw-managed images, allowing recovery onto the current managed image. The normalized names must exactly match the installer's printed array. Set it only after verifying every named sandbox. Recorded custom-image evidence remains blocked. |
| `NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR` | `1` to enable | Skips the automatic DNS-proxy repair for stale `inference.local` routes during `$$nemoclaw <name> connect` and `$$nemoclaw <name> connect --probe-only`. Use only as a troubleshooting escape hatch. |
| `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. |
Expand Down
2 changes: 1 addition & 1 deletion src/commands/sandbox/destroy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export default class DestroyCliCommand extends NemoClawCommand {
force: forceFlag(),
"cleanup-gateway": Flags.boolean({
description:
"When destroying the last sandbox, also tear down the shared NemoClaw gateway. Default: preserve. NEMOCLAW_CLEANUP_GATEWAY=1 sets the same default.",
"When destroying the last sandbox, also tear down the shared NemoClaw gateway. Default: preserve on Linux; cleanup for unattended macOS destroys. NEMOCLAW_CLEANUP_GATEWAY overrides the platform default.",
allowNo: true,
}),
};
Expand Down
25 changes: 25 additions & 0 deletions src/lib/actions/sandbox/destroy-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ describe("destroySandbox flow", () => {
? delete process.env.OPENSHELL_GATEWAY
: (process.env.OPENSHELL_GATEWAY = originalGatewayEnv);
vi.restoreAllMocks();
vi.unstubAllEnvs();
resetDestroyModuleCache();
});

Expand All @@ -54,6 +55,30 @@ describe("destroySandbox flow", () => {
expectSuccessfulLiveDestroy(harness, exitSpy);
});

it.each([
["--yes", "darwin", { yes: true }, "", true],
["NEMOCLAW_NON_INTERACTIVE=1", "darwin", {}, "1", true],
[
"an explicit preservation override",
"darwin",
{ yes: true, cleanupGateway: false },
"",
false,
],
["NEMOCLAW_NON_INTERACTIVE=1", "linux", {}, "1", false],
] as const)("applies the final-gateway default for %s on %s (#4662)", async (_scenario, platform, options, nonInteractive, cleanupExpected) => {
vi.spyOn(process, "platform", "get").mockReturnValue(platform);
vi.stubEnv("NEMOCLAW_NON_INTERACTIVE", nonInteractive);
const harness = createDestroyHarness();

await expect(harness.destroySandbox("alpha", options)).resolves.toBeUndefined();

expect(harness.promptSpy).not.toHaveBeenCalled();
expect(harness.cleanupGatewaySpy.mock.calls).toEqual(
cleanupExpected ? [["nemoclaw-19080", harness.runOpenshellSpy]] : [],
);
});

it("stops before local cleanup when OpenShell fails to delete the live sandbox", async () => {
const harness = createDestroyHarness({
deleteStatus: 7,
Expand Down
133 changes: 133 additions & 0 deletions src/lib/actions/sandbox/destroy-gateway-cleanup.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import { hasNoLiveSandboxes } from "../../domain/sandbox/destroy";
import {
collectLiveSandboxProbeSnapshot,
shouldCleanupGatewayAfterConfirmedFinalDestroy,
} from "./destroy-gateway-cleanup";

describe("shouldCleanupGatewayAfterConfirmedFinalDestroy", () => {
it("defers live probes until the local registry is empty", () => {
const liveSandboxProbe = vi.fn(() => true);

expect(
shouldCleanupGatewayAfterConfirmedFinalDestroy(
{
deleteSucceededOrAlreadyGone: true,
removedRegistryEntry: true,
},
{
listSandboxes: () => ({ sandboxes: [{}] }),
liveSandboxProbe,
},
),
).toBe(false);
expect(liveSandboxProbe).not.toHaveBeenCalled();
});

it("requires confirmed delete, registry removal, and no live sandboxes", () => {
expect(
shouldCleanupGatewayAfterConfirmedFinalDestroy(
{
deleteSucceededOrAlreadyGone: true,
removedRegistryEntry: true,
},
{
listSandboxes: () => ({ sandboxes: [] }),
liveSandboxProbe: () => true,
},
),
).toBe(true);

expect(
shouldCleanupGatewayAfterConfirmedFinalDestroy(
{
deleteSucceededOrAlreadyGone: true,
removedRegistryEntry: true,
},
{
listSandboxes: () => ({ sandboxes: [] }),
liveSandboxProbe: () => false,
},
),
).toBe(false);
});

it("preserves the gateway when a live sandbox appears after the empty-registry check", () => {
const events: string[] = [];
expect(
shouldCleanupGatewayAfterConfirmedFinalDestroy(
{
deleteSucceededOrAlreadyGone: true,
removedRegistryEntry: true,
},
{
listSandboxes: () => {
events.push("registry-empty");
return { sandboxes: [] };
},
liveSandboxProbe: () => {
events.push("live-sandbox-observed");
// False means the host probe observed a sandbox during the TOCTOU window.
return false;
},
},
),
).toBe(false);
expect(events).toEqual(["registry-empty", "live-sandbox-observed"]);
});

it("collects OpenShell and Docker live-sandbox snapshots in the action layer", () => {
const captureOpenshell = vi.fn(() => ({
status: 0,
output:
"NAME CREATED PHASE\nnpmtest now Error\n",
}));
const dockerCapture = vi.fn(() => "openshell-npmtest-e487d1bd\n");

const snapshot = collectLiveSandboxProbeSnapshot({
captureOpenshell,
dockerCapture,
timeoutMs: 1_000,
});

expect(captureOpenshell).toHaveBeenCalledWith(["sandbox", "list"], {
ignoreError: true,
timeout: 1_000,
});
expect(dockerCapture).toHaveBeenCalledWith(
["ps", "--filter", "name=openshell-npmtest-", "--format", "{{.Names}}"],
{
timeout: 1_000,
},
);
expect(hasNoLiveSandboxes(snapshot)).toBe(false);
});

it("records failed Docker probes as fail-closed snapshots", () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
const snapshot = collectLiveSandboxProbeSnapshot({
captureOpenshell: () => ({
status: 0,
output:
"NAME CREATED PHASE\nnpmtest now Failed\n",
}),
dockerCapture: () => {
throw new Error("docker unavailable");
},
timeoutMs: 1_000,
});

expect(hasNoLiveSandboxes(snapshot)).toBe(false);
expect(snapshot.dockerContainersBySandboxName.get("npmtest")).toEqual({
output: "",
probeFailed: true,
});
expect(warn).toHaveBeenCalledWith(
"Docker container probe failed for sandbox 'npmtest'; preserving shared gateway: Error: docker unavailable",
);
});
});
Loading
Loading