Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/lib/build-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,22 @@ describe("printSandboxCreateRecoveryHints", () => {
expect(out).toContain("NEMOCLAW_WEB_SEARCH_ENABLED=0");
expect(out).toContain("onboard --resume");
});

it("prints the portable-profile recovery command when the portable env is set", () => {
const prev = process.env.NEMOCLAW_EXPERIMENTAL_PROFILE;
process.env.NEMOCLAW_EXPERIMENTAL_PROFILE = "portable";
try {
printSandboxCreateRecoveryHints("");
const out = stderr();
expect(out).toContain("onboard --experimental-profile portable");
expect(out).not.toContain("--resume");
expect(out).not.toContain("Or: nemoclaw onboard");
} finally {
prev === undefined
? delete process.env.NEMOCLAW_EXPERIMENTAL_PROFILE
: (process.env.NEMOCLAW_EXPERIMENTAL_PROFILE = prev);
}
});
});

describe("reconstructImageRefCreateCommand", () => {
Expand Down
27 changes: 15 additions & 12 deletions src/lib/build-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
import fs from "node:fs";
import path from "node:path";
import { CLI_NAME } from "./cli/branding";
import { noteOnboardResumeHintShown } from "./onboard/resume-hint";
import { isPortableExperimentalProfile } from "./onboard/experimental/portable-profile";
import { noteOnboardResumeHintShown, onboardRecoveryCommand } from "./onboard/resume-hint";

import { classifySandboxCreateFailure, planSandboxCreateRecovery } from "./validation";

Expand Down Expand Up @@ -107,6 +108,8 @@ export function printSandboxCreateRecoveryHints(
// Every branch below prints tailored `--resume` recovery guidance, so suppress
// the generic incomplete-exit backstop (#6003).
noteOnboardResumeHintShown();
const portable = isPortableExperimentalProfile();
const recoveryCommand = onboardRecoveryCommand(portable);
const failure = classifySandboxCreateFailure(output);
if (failure.kind === "image_upload_container_missing") {
const { arm64ImageRefWorkaround } = planSandboxCreateRecovery(failure, { platform, arch });
Expand Down Expand Up @@ -173,14 +176,14 @@ export function printSandboxCreateRecoveryHints(
);
}
console.error(
` If you would rather let NemoClaw rebuild and retry from scratch: ${CLI_NAME} onboard --resume`,
` If you would rather let NemoClaw rebuild and retry from scratch: ${recoveryCommand}`,
);
return;
}
if (failure.kind === "image_transfer_timeout") {
console.error(" Hint: image upload into the OpenShell gateway timed out.");
console.error(` Recovery: ${CLI_NAME} onboard --resume`);
if (failure.uploadedToGateway) {
console.error(` Recovery: ${recoveryCommand}`);
if (failure.uploadedToGateway && !portable) {
console.error(
" Progress reached the gateway upload stage, so resume may be able to reuse existing gateway state.",
);
Expand All @@ -190,7 +193,7 @@ export function printSandboxCreateRecoveryHints(
}
if (failure.kind === "image_transfer_reset") {
console.error(" Hint: the image push/import stream was interrupted.");
console.error(` Recovery: ${CLI_NAME} onboard --resume`);
console.error(` Recovery: ${recoveryCommand}`);
if (failure.uploadedToGateway) {
console.error(" The image appears to have reached the gateway before the stream failed.");
}
Expand All @@ -199,7 +202,7 @@ export function printSandboxCreateRecoveryHints(
}
if (failure.kind === "sandbox_create_incomplete") {
console.error(" Hint: sandbox creation started but the create stream did not finish cleanly.");
console.error(` Recovery: ${CLI_NAME} onboard --resume`);
console.error(` Recovery: ${recoveryCommand}`);
console.error(
" Check: openshell sandbox list # verify whether the sandbox became ready",
);
Expand All @@ -210,7 +213,7 @@ export function printSandboxCreateRecoveryHints(
" Hint: TLS certificate mismatch — the gateway certificate changed since the CLI last trusted it.",
);
console.error(" Fix: openshell gateway trust -g nemoclaw");
console.error(` Then: ${CLI_NAME} onboard --resume`);
console.error(` Then: ${recoveryCommand}`);
return;
}
if (failure.kind === "gpu_cdi_injection_failed") {
Expand All @@ -220,9 +223,9 @@ export function printSandboxCreateRecoveryHints(
);
console.error(" NEMOCLAW_DOCKER_GPU_PATCH=0 does not bypass this path.");
console.error(" Skip GPU passthrough entirely with either:");
console.error(` ${CLI_NAME} onboard --no-gpu`);
console.error(` ${portable ? recoveryCommand : `${CLI_NAME} onboard`} --no-gpu`);
console.error(" NEMOCLAW_SANDBOX_GPU=0 (env var, applies to subsequent runs)");
console.error(` Recovery: ${CLI_NAME} onboard --resume --no-gpu`);
if (!portable) console.error(` Recovery: ${recoveryCommand} --no-gpu`);
return;
}
if (failure.kind === "plugin_install_network_denied") {
Expand All @@ -237,9 +240,9 @@ export function printSandboxCreateRecoveryHints(
console.error(
" feature that requires this plugin (e.g. NEMOCLAW_WEB_SEARCH_ENABLED=0).",
);
console.error(` Recovery: ${CLI_NAME} onboard --resume`);
console.error(` Recovery: ${recoveryCommand}`);
return;
}
console.error(` Recovery: ${CLI_NAME} onboard --resume`);
console.error(` Or: ${CLI_NAME} onboard`);
console.error(` Recovery: ${recoveryCommand}`);
if (!portable) console.error(` Or: ${CLI_NAME} onboard`);
}
10 changes: 8 additions & 2 deletions src/lib/onboard/exit-step-failure.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,9 @@ describe("terminal step failure helper", () => {
expect(loaded.machine.state).toBe("failed");
});

it("simulates the onboard exit listener and ignores successful or complete exits", () => {
it("keeps the portable profile captured by the onboard exit listener (#8873)", () => {
const listeners: Array<(code: number) => void> = [];
const errors: string[] = [];
let complete = false;
const processLike = {
once: (event: "exit", listener: (code: number) => void) => {
Expand All @@ -82,13 +83,16 @@ describe("terminal step failure helper", () => {
};
session.saveSession(session.createSession({ lastStepStarted: "inference" }));
// The incomplete exit also prints the #6003 resume hint; capture it.
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const errorSpy = vi
.spyOn(console, "error")
.mockImplementation((message = "") => errors.push(String(message)));

registerIncompleteOnboardExitFailureHandler(
session,
() => complete,
"Onboarding exited before the step completed.",
processLike,
true,
);
listeners[0](0);
expect(requireLoadedSession().status).toBe("in_progress");
Expand All @@ -100,6 +104,8 @@ describe("terminal step failure helper", () => {
complete = false;
listeners[0](1);
errorSpy.mockRestore();
expect(errors.join("\n")).toContain("onboard --experimental-profile portable");
expect(errors.join("\n")).not.toContain("onboard --resume");

const loaded = requireLoadedSession();
expect(loaded.steps.inference.status).toBe("failed");
Expand Down
12 changes: 7 additions & 5 deletions src/lib/onboard/exit-step-failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

import type { Session } from "../state/onboard-session";
import { isPortableExperimentalProfile } from "./experimental/portable-profile";
import { printOnboardResumeHint } from "./resume-hint";

export interface ExitStepFailureSessionDeps {
Expand Down Expand Up @@ -44,17 +45,18 @@ export function registerIncompleteOnboardExitFailureHandler(
isComplete: () => boolean,
message: string,
processLike: OnboardExitFailureProcessLike = process,
portable = isPortableExperimentalProfile(),
): void {
const failIncompleteStep = (): void => {
if (isComplete()) return;
// A non-null return means a step was in progress, so the session records a
// resumable point — surface `--resume` for exit paths that don't print
// their own recovery guidance (#6003). When an explicit cancel has already
// cleared the session (or no step started), this is null and stays silent;
// A non-null return means a step was in progress, so surface the
// profile-appropriate recovery command for exit paths that don't print
// their own guidance (#6003, #8873). When an explicit cancel has already
// cleared the session (or no step started), this is null and stays silent.
// printOnboardResumeHint also self-dedupes against tailored hints.
const interrupted = markLastStartedStepFailed(deps, message, true);
if (!interrupted) return;
printOnboardResumeHint();
printOnboardResumeHint(portable);
};

processLike.once("exit", (code) => {
Expand Down
12 changes: 12 additions & 0 deletions src/lib/onboard/gateway-start-failure-integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,18 @@ describe("startGatewayWithOptions docker-unreachable abort (#2347)", () => {
expect(joined).not.toContain("colima start");
expect(joined).not.toContain("systemctl");
});

it("prints the rootless-Podman recovery hint when portable=true (#8873)", () => {
const printed: string[] = [];
printDockerDaemonRecovery((message = "") => printed.push(message), "linux", true);
const joined = printed.join("\n");
expect(joined).toContain("rootless Podman API service is not reachable");
expect(joined).toContain("Start Podman");
expect(joined).toContain("nemoclaw onboard --experimental-profile portable");
expect(joined).not.toContain("sudo systemctl start docker");
expect(joined).not.toContain("colima start");
expect(joined).not.toContain("--resume");
});
});

// ── Layer 1: handleFinalGatewayStartFailure dockerUnreachable branch ─────
Expand Down
12 changes: 11 additions & 1 deletion src/lib/onboard/gateway-start-failure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import { compactText } from "../core/url-utils";
import { redact } from "../security/redact";
import { classifyGatewayStartFailure } from "../validation";
import { isPortableExperimentalProfile } from "./experimental/portable-profile";
import { onboardRecoveryCommand } from "./resume-hint";

const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g;

Expand Down Expand Up @@ -45,7 +47,15 @@ export function reportLegacyGatewayStartResultFailure(
export function printDockerDaemonRecovery(
printError: (message?: string) => void,
platform: NodeJS.Platform = process.platform,
portable = isPortableExperimentalProfile(),
): void {
if (portable) {
printError(" The rootless Podman API service is not reachable.");
printError("");
printError(` Start Podman, then rerun: ${onboardRecoveryCommand(portable)}`);
return;
}

printError(" Docker daemon is not running — cannot start the gateway.");
printError("");
printError(" Start Docker, then rerun `nemoclaw onboard`:");
Expand Down Expand Up @@ -116,7 +126,7 @@ export function createFinalGatewayStartFailureHandler(deps: FinalGatewayStartFai
printError(
` docker volume ls -q --filter "name=openshell-cluster-${gatewayName}" | xargs -r docker volume rm`,
);
printError(" nemoclaw onboard --resume");
printError(` ${onboardRecoveryCommand()}`);
return exitProcess(1);
};
}
38 changes: 34 additions & 4 deletions src/lib/onboard/resume-hint.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,55 @@ beforeEach(() => resetOnboardResumeHintForTests());
afterEach(() => resetOnboardResumeHintForTests());

describe("onboard resume hint", () => {
const portableEnv = "NEMOCLAW_EXPERIMENTAL_PROFILE";

it("prints the --resume recovery guidance through the injected logger", () => {
const lines: string[] = [];
printOnboardResumeHint((message) => lines.push(message));
printOnboardResumeHint(false, (message) => lines.push(message));
const text = lines.join("\n");
expect(text).toContain("onboard --resume");
expect(text).toContain("--fresh");
});

it("prints the portable-profile recovery guidance when the portable env is set (#8873)", () => {
const prev = process.env[portableEnv];
process.env[portableEnv] = "portable";
try {
const lines: string[] = [];
printOnboardResumeHint(undefined, (message) => lines.push(message));
const text = lines.join("\n");
expect(text).toContain("onboard --experimental-profile portable");
expect(text).not.toContain("--resume");
} finally {
prev === undefined ? delete process.env[portableEnv] : (process.env[portableEnv] = prev);
}
});

it("uses an explicit portable profile after the environment is restored (#8873)", () => {
const prev = process.env[portableEnv];
delete process.env[portableEnv];
try {
const lines: string[] = [];
printOnboardResumeHint(true, (message) => lines.push(message));
const text = lines.join("\n");
expect(text).toContain("onboard --experimental-profile portable");
expect(text).not.toContain("--resume");
} finally {
prev === undefined ? delete process.env[portableEnv] : (process.env[portableEnv] = prev);
}
});

it("prints at most once per process", () => {
const lines: string[] = [];
printOnboardResumeHint((message) => lines.push(message));
printOnboardResumeHint((message) => lines.push(message));
printOnboardResumeHint(false, (message) => lines.push(message));
printOnboardResumeHint(false, (message) => lines.push(message));
expect(lines.filter((line) => line.includes("onboard --resume"))).toHaveLength(1);
});

it("stays silent once a tailored hint was noted", () => {
const lines: string[] = [];
noteOnboardResumeHintShown();
printOnboardResumeHint((message) => lines.push(message));
printOnboardResumeHint(false, (message) => lines.push(message));
expect(lines).toHaveLength(0);
});
});
29 changes: 22 additions & 7 deletions src/lib/onboard/resume-hint.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@
// SPDX-License-Identifier: Apache-2.0

import { CLI_NAME } from "../cli/branding";
import { isPortableExperimentalProfile } from "./experimental/portable-profile";

export function onboardRecoveryCommand(portable = isPortableExperimentalProfile()): string {
return portable
? `${CLI_NAME} onboard --experimental-profile portable`
: `${CLI_NAME} onboard --resume`;
}

// Whether an onboard `--resume` recovery hint has already been emitted this run.
// Context-specific failure explainers (e.g. the sandbox build-context hints)
Expand All @@ -11,27 +18,35 @@ import { CLI_NAME } from "../cli/branding";
let resumeHintShown = false;

/**
* Print the generic onboard `--resume` recovery hint, once per process.
* Print the generic onboard recovery hint, once per process.
*
* Onboarding exits through dozens of scattered `process.exit(1)` paths; most
* never mention `--resume`, so users assume a failed run requires a full
* never mention how to resume, so users assume a failed run requires a full
* reinstall (#6003). The incomplete-exit handler calls this as a catch-all when
* a resumable step was in progress, covering every exit that does not already
* print its own recovery guidance.
* print its own recovery guidance. The recovery command adapts to whether the
* run selected the portable experimental profile (which forces `--fresh` and
* rejects `--resume`) (#8873).
*/
export function printOnboardResumeHint(
portable = isPortableExperimentalProfile(),
log: (message: string) => void = (message) => console.error(message),
): void {
if (resumeHintShown) return;
resumeHintShown = true;
log("");
log(" Onboarding did not finish. Resume from the step that failed with:");
log(` ${CLI_NAME} onboard --resume`);
log(" Completed steps are skipped; pass --fresh instead to start over.");
if (portable) {
log(" Onboarding did not finish. Portable onboarding always starts fresh; rerun:");
log(` ${onboardRecoveryCommand(portable)}`);
} else {
log(" Onboarding did not finish. Resume from the step that failed with:");
log(` ${onboardRecoveryCommand(portable)}`);
log(" Completed steps are skipped; pass --fresh instead to start over.");
}
}

/**
* Record that a context-specific `--resume` hint was already printed this run so
* Record that a context-specific hint was already printed this run so
* the catch-all in {@link printOnboardResumeHint} stays silent.
*/
export function noteOnboardResumeHintShown(): void {
Expand Down
Loading