Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/managed-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ on:
- "src/lib/state/paths.ts"
- "src/lib/state/state-root.ts"
- "test/e2e/live/mcp-bridge*.ts"
- "test/e2e/fixtures/phases/lifecycle.ts"
- "test/e2e/live/managed-image-activation-e2e*.ts"
- "src/lib/tool-disclosure.ts"
- "tools/mcp-tool-discovery-runtime/**"
Expand Down
28 changes: 6 additions & 22 deletions test/e2e/fixtures/phases/lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -734,33 +734,17 @@ export class LifecyclePhaseFixture {
previousRuntime: HostGatewayRuntime | null,
options: { requireUserService?: boolean; sandboxName?: string } = {},
): Promise<ShellProbeResult> {
if (options.sandboxName && options.requireUserService !== true) {
return await this.host.nemoclaw([options.sandboxName, "status"], {
artifactName: `lifecycle-gateway-recover-through-nemoclaw-status-${options.sandboxName}`,
env: buildAvailabilityProbeEnv(),
timeoutMs: 120_000,
});
}
const userServiceStart = await this.startOpenShellGatewayUserService({
requireAvailable: options.requireUserService,
});
if (userServiceStart) return userServiceStart;
if (options.sandboxName) {
return await this.host.nemoclaw([options.sandboxName, "status"], {
artifactName: `lifecycle-gateway-recover-through-nemoclaw-status-${options.sandboxName}`,
env: buildAvailabilityProbeEnv(),
timeoutMs: 120_000,
});
}
if (previousRuntime?.kind === "pid") {
return await this.host.nemoclaw(["status"], {
artifactName: "lifecycle-gateway-recover-through-nemoclaw-status",
env: buildAvailabilityProbeEnv(),
timeoutMs: 120_000,
});
if (!options.sandboxName) {
throw new Error(
`A sandbox name is required to recover the stopped ${previousRuntime?.kind ?? "unknown"} gateway runtime.`,
);
}
return await this.host.command("openshell", ["gateway", "start", "--name", "nemoclaw"], {
artifactName: "lifecycle-gateway-start",
return await this.host.nemoclaw([options.sandboxName, "recover"], {
artifactName: `lifecycle-gateway-recover-through-nemoclaw-recover-${options.sandboxName}`,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
env: buildAvailabilityProbeEnv(),
timeoutMs: 120_000,
});
Expand Down
53 changes: 39 additions & 14 deletions test/e2e/support/e2e-phase-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,7 @@ describe("LifecyclePhaseFixture rebuild helpers", () => {
});

describe("LifecyclePhaseFixture gateway runtime restart helpers", () => {
it("falls back to PID/container controls when the selected user service is inactive (#10947)", async () => {
it("restores a PID gateway before sandbox recovery when the user service is inactive (#11640)", async () => {
const runner = new FakeRunner();
runner.enqueue(shellResult(0, "12345\n")); // resolveHostRuntime pid probe
runner.enqueue(shellResult(0)); // forward stop
Expand All @@ -538,17 +538,16 @@ describe("LifecyclePhaseFixture gateway runtime restart helpers", () => {
runner.enqueue(shellResult(1, "")); // expectHostRuntimeStopped pid probe
runner.enqueue(shellResult(0, "")); // expectHostRuntimeStopped container probe
runner.enqueue(shellResult(0)); // lifecycle-gateway-stopped true artifact
runner.enqueue(shellResult(0, "status recovered\n")); // start through nemoclaw status
runner.enqueue(shellResult(0, "gateway recovered\n")); // start through sandbox recovery
runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // waitForGatewayConnected
const cleanup = new FakeCleanup();
const host = new HostCliClient(runner);
const sandbox = new SandboxClient(runner);
const fx = new LifecyclePhaseFixture(host, sandbox, cleanup, new GatewayClient(host, sandbox));

await expect(fx.restartGatewayRuntime({ delayMs: 0 })).resolves.toEqual({
kind: "pid",
id: "12345",
});
await expect(
fx.restartGatewayRuntime({ delayMs: 0, sandboxName: "e2e-survival" }),
).resolves.toEqual({ kind: "pid", id: "12345" });
await fx.waitForGatewayConnected({ attempts: 1, intervalMs: 1 });

expect(runner.calls.map((call) => `${call.command} ${call.args.join(" ")}`)).toEqual([
Expand All @@ -561,7 +560,7 @@ describe("LifecyclePhaseFixture gateway runtime restart helpers", () => {
expect.stringContaining("sh -lc pid_file="),
"docker container ps --format {{.ID}}\t{{.Names}}",
"true ",
"nemoclaw status",
"nemoclaw e2e-survival recover",
"openshell status",
]);
});
Expand Down Expand Up @@ -679,24 +678,50 @@ describe("LifecyclePhaseFixture gateway runtime restart helpers", () => {
expect(runner.calls).toHaveLength(2);
});

it("can recover a PID runtime through sandbox-specific status", async () => {
it("restores a container runtime through sandbox-specific recovery", async () => {
const runner = new FakeRunner();
runner.enqueue(shellResult(0, "status recovered\n"));
runner.enqueue(shellResult(0, "gateway recovered\n"));
const cleanup = new FakeCleanup();

await expect(
fixture(runner, cleanup).startGatewayRuntime(
{ kind: "pid", id: "12345" },
{
sandboxName: "e2e-survival",
},
{ kind: "container", id: "container-1" },
{ sandboxName: "e2e-survival" },
),
).resolves.toMatchObject({ exitCode: 0 });

expect(runner.calls.map((call) => `${call.command} ${call.args.join(" ")}`)).toEqual([
"nemoclaw e2e-survival status",
"nemoclaw e2e-survival recover",
]);
});

it("restarts the stopped user service before sandbox recovery", async () => {
const runner = new FakeRunner();
const cleanup = new FakeCleanup();
const fx = fixture(runner, cleanup);
runner.enqueue(shellResult(0)); // forward stop
runner.enqueue(shellResult(0, stoppedGatewayUserService)); // user service stop

await fx.stopGatewayRuntime();

runner.enqueue(shellResult(0)); // selected user service restart
await fx.startGatewayRuntime({ kind: "pid", id: "12345" }, { sandboxName: "e2e-survival" });

expect(runner.calls.map((call) => call.options?.artifactName)).toEqual([
"lifecycle-gateway-forward-stop",
"lifecycle-gateway-user-service-stop",
"lifecycle-gateway-user-service-restart",
]);
});

it("fails closed without a sandbox recovery target when no user service owns the gateway", async () => {
await expect(
fixture(new FakeRunner(), new FakeCleanup()).startGatewayRuntime({
kind: "pid",
id: "12345",
}),
).rejects.toThrow(/sandbox name is required to recover the stopped pid gateway runtime/i);
});
});

describe("LifecyclePhaseFixture profile dispatch", () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,7 @@ describe("complete managed-image publication workflow", () => {
expect(workflow.on?.pull_request?.paths).toEqual(
expect.arrayContaining([
"src/lib/onboard/**",
"test/e2e/fixtures/phases/lifecycle.ts",
"test/e2e/live/managed-image-activation-e2e*.ts",
]),
);
Expand Down
Loading