Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ const {
getProviderSelectionConfig,
parseGatewayInference,
} = inferenceConfig;
const { ensureResumeProviderReady } = require("./onboard/resume-provider-shim");

const onboardProviders = require("./onboard/providers");
const hermesProviderAuth = require("./hermes-provider-auth");
Expand Down Expand Up @@ -9506,18 +9507,17 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
let webSearchConfig = session?.webSearchConfig || null;
let forceProviderSelection = forceProviderSelectionForAgentChange;
while (true) {
let forceInferenceSetup = false;
const resumeProviderSelection =
!forceProviderSelection &&
resume &&
session?.steps?.provider_selection?.status === "complete" &&
typeof provider === "string" &&
typeof model === "string";
if (resumeProviderSelection) {
({ forceInferenceSetup } = await ensureResumeProviderReady(provider, credentialEnv));
skippedStepMessage("provider_selection", `${provider} / ${model}`);
hydrateCredentialEnv(credentialEnv);
// #3342: resume short-circuits provider selection — repair the
// ollama-local systemd loopback override here so legacy 0.0.0.0
// drop-ins from older NemoClaw versions get rewritten every resume.
repairLocalInferenceSystemdOverrideOrExit(provider, isNonInteractive);
} else {
// #2753: do not persist sandboxName to onboard-session.json before
Expand Down Expand Up @@ -9561,6 +9561,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
const resumeInference =
!needsBedrockRuntimeAdapter &&
!forceProviderSelection &&
!forceInferenceSetup &&
resume &&
isInferenceRouteReady(provider, model);
if (resumeInference) {
Expand Down Expand Up @@ -10104,6 +10105,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
module.exports = {
buildOrphanedSandboxRollbackMessage,
buildProviderArgs,
resumeProviderShimDeps: { isRoutedInferenceProvider, replaceNamedCredential },
buildGatewayBootstrapSecretsScript,
buildCompatibleEndpointSandboxSmokeCommand,
buildCompatibleEndpointSandboxSmokeScript,
Expand Down
129 changes: 129 additions & 0 deletions src/lib/onboard/resume-provider-recovery.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";

import {
ensureResumeProviderReady,
type RemoteProviderConfigEntry,
type ResumeProviderRecoveryDeps,
} from "./resume-provider-recovery";

const COMPATIBLE_ENDPOINT_CONFIG: RemoteProviderConfigEntry = {
label: "Compatible Endpoint",
providerName: "compatible-endpoint",
providerType: "openai",
credentialEnv: "COMPATIBLE_API_KEY",
endpointUrl: "https://example/v1",
helpUrl: null,
modelMode: "input",
defaultModel: "test-model",
};

type DepsRecorder = {
log: string[];
warn: string[];
note: string[];
exitCalls: number[];
replaceCalls: Array<{ env: string; label: string }>;
deps: ResumeProviderRecoveryDeps;
};

function makeDeps(overrides: {
providerExists?: boolean;
credentialValue?: string | null;
nonInteractive?: boolean;
remoteProviderConfig?: Record<string, RemoteProviderConfigEntry>;
}): DepsRecorder {
const log: string[] = [];
const warn: string[] = [];
const note: string[] = [];
const exitCalls: number[] = [];
const replaceCalls: Array<{ env: string; label: string }> = [];
const deps: ResumeProviderRecoveryDeps = {
remoteProviderConfig: overrides.remoteProviderConfig ?? {
compatible: COMPATIBLE_ENDPOINT_CONFIG,
},
defaultRouteCredentialEnv: "OPENAI_API_KEY",
isRoutedInferenceProvider: () => false,
providerExistsInGateway: () => overrides.providerExists ?? true,
hydrateCredentialEnv: () => overrides.credentialValue ?? null,
getProviderLabel: (key) => key,
isNonInteractive: () => overrides.nonInteractive ?? false,
log: (m) => log.push(m),
warn: (m) => warn.push(m),
note: (m) => note.push(m),
exit: (code) => exitCalls.push(code),
replaceNamedCredential: async (env, label) => {
replaceCalls.push({ env, label });
return "fresh-key";
},
validateNvidiaApiKeyValue: () => null,
};
return { log, warn, note, exitCalls, replaceCalls, deps };
}

describe("ensureResumeProviderReady", () => {
it("returns false-forced when no provider is set (nothing to recover)", async () => {
const { deps } = makeDeps({ providerExists: false });
const result = await ensureResumeProviderReady(null, null, deps);
expect(result.forceInferenceSetup).toBe(false);
});

it("returns false-forced when the provider is unknown and not a routed provider", async () => {
const { deps } = makeDeps({ providerExists: false });
const result = await ensureResumeProviderReady("mystery-provider", null, deps);
expect(result.forceInferenceSetup).toBe(false);
});

it("returns false-forced when the provider still exists in the gateway", async () => {
const { deps } = makeDeps({ providerExists: true });
const result = await ensureResumeProviderReady("compatible-endpoint", "COMPATIBLE_API_KEY", deps);
expect(result.forceInferenceSetup).toBe(false);
});

it("emits a [resume] note and forces inference setup when credential is already hydrated", async () => {
const recorder = makeDeps({
providerExists: false,
credentialValue: "already-hydrated-key",
});
const result = await ensureResumeProviderReady(
"compatible-endpoint",
"COMPATIBLE_API_KEY",
recorder.deps,
);
expect(result.forceInferenceSetup).toBe(true);
expect(recorder.note.join("\n")).toContain("[resume]");
expect(recorder.replaceCalls).toHaveLength(0);
});

it("re-prompts for credentials when the provider was reset and credential is missing (#3278)", async () => {
const recorder = makeDeps({
providerExists: false,
credentialValue: null,
});
const result = await ensureResumeProviderReady(
"compatible-endpoint",
"COMPATIBLE_API_KEY",
recorder.deps,
);
expect(result.forceInferenceSetup).toBe(true);
expect(recorder.replaceCalls).toEqual([
{ env: "COMPATIBLE_API_KEY", label: "Compatible Endpoint API key" },
]);
expect(recorder.exitCalls).toEqual([]);
});

it("exits 1 in non-interactive mode when the provider is missing and no credential is set", async () => {
const recorder = makeDeps({
providerExists: false,
credentialValue: null,
nonInteractive: true,
});
await ensureResumeProviderReady("compatible-endpoint", "COMPATIBLE_API_KEY", recorder.deps);
expect(recorder.exitCalls).toEqual([1]);
expect(recorder.warn.join("\n")).toContain("COMPATIBLE_API_KEY");
expect(recorder.warn.join("\n")).toContain("during resume");
expect(recorder.replaceCalls).toHaveLength(0);
});
});
123 changes: 123 additions & 0 deletions src/lib/onboard/resume-provider-recovery.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Re-prompts for a remote provider's credential during `nemoclaw onboard --resume`
// when the previously-recorded provider has been deleted from the gateway (e.g.
// after `nemoclaw credentials reset <PROVIDER>` removed it).
//
// Resume mode would otherwise short-circuit the inference setup step on the
// recorded `provider`/`model`, leaving the sandbox rebuild to fail with an
// authentication error (#3278).

export type RemoteProviderConfigEntry = {
label: string;
providerName: string;
providerType: string;
credentialEnv: string;
endpointUrl: string;
helpUrl: string | null;
modelMode: "catalog" | "curated" | "input";
defaultModel: string;
skipVerify?: boolean;
};

export type ResumeProviderRecoveryDeps = {
remoteProviderConfig: Record<string, RemoteProviderConfigEntry>;
defaultRouteCredentialEnv: string;
isRoutedInferenceProvider: (provider: string) => boolean;
providerExistsInGateway: (name: string) => boolean;
hydrateCredentialEnv: (envName: string) => string | null;
getProviderLabel: (key: string) => string;
isNonInteractive: () => boolean;
log: (message: string) => void;
warn: (message: string) => void;
note: (message: string) => void;
exit: (code: number) => void;
replaceNamedCredential: (
envName: string,
label: string,
helpUrl: string | null,
validator: (value: string) => string | null,
) => Promise<unknown>;
validateNvidiaApiKeyValue: (key: string, credentialEnv: string) => string | null;
};

/**
* Resolve a persisted OpenShell provider name back to its onboard provider config.
*/
export function getRemoteProviderConfigForName(
provider: string | null | undefined,
remoteProviderConfig: Record<string, RemoteProviderConfigEntry>,
): RemoteProviderConfigEntry | null {
if (!provider) return null;
if (provider === "nvidia-nim") return remoteProviderConfig.build;
return (
Object.values(remoteProviderConfig).find((entry) => entry.providerName === provider) || null
);
}

/**
* Choose the credential env used to recreate a missing provider during resume.
*/
export function getResumeProviderCredentialEnv(
provider: string,
config: RemoteProviderConfigEntry | null,
credentialEnv: string | null | undefined,
deps: Pick<ResumeProviderRecoveryDeps, "defaultRouteCredentialEnv" | "isRoutedInferenceProvider">,
): string {
if (credentialEnv) return credentialEnv;
if (config?.credentialEnv) return config.credentialEnv;
return deps.isRoutedInferenceProvider(provider) ? deps.defaultRouteCredentialEnv : "";
}

/**
* Ensure a resumed remote provider still exists in the gateway, re-prompting
* for credentials when needed.
*
* Returns `{ forceInferenceSetup: true }` when the caller must re-run the
* inference setup step (provider was missing and credential was hydrated or
* just re-entered). Returns `{ forceInferenceSetup: false }` when no action
* is needed.
*
* In non-interactive mode with a missing credential, calls `deps.exit(1)`.
*/
export async function ensureResumeProviderReady(
provider: string | null | undefined,
credentialEnv: string | null | undefined,
deps: ResumeProviderRecoveryDeps,
): Promise<{ forceInferenceSetup: boolean }> {
const config = getRemoteProviderConfigForName(provider, deps.remoteProviderConfig);
if (!provider || (!config && !deps.isRoutedInferenceProvider(provider))) {
return { forceInferenceSetup: false };
}
if (deps.providerExistsInGateway(provider)) return { forceInferenceSetup: false };

const resolvedCredentialEnv = getResumeProviderCredentialEnv(provider, config, credentialEnv, deps);
const credentialValue = deps.hydrateCredentialEnv(resolvedCredentialEnv);
const providerLabel = config?.label || deps.getProviderLabel(provider) || provider;
const helpUrl = config?.helpUrl || null;
if (!credentialValue) {
if (deps.isNonInteractive()) {
deps.warn(
` ${resolvedCredentialEnv} is required to recreate provider '${provider}' during resume.`,
);
deps.warn(
` Re-run without --non-interactive to enter it, or set ${resolvedCredentialEnv} and retry.`,
);
deps.exit(1);
return { forceInferenceSetup: false };
}
deps.log("");
deps.log(` [resume] Provider '${provider}' is missing from the gateway.`);
deps.log(" Re-enter the API key so onboarding can recreate it before rebuilding.");
await deps.replaceNamedCredential(
resolvedCredentialEnv,
`${providerLabel} API key`,
helpUrl,
(value) => deps.validateNvidiaApiKeyValue(value, resolvedCredentialEnv),
);
} else {
deps.note(` [resume] Provider '${provider}' is missing from the gateway; recreating it.`);
}
return { forceInferenceSetup: true };
}
54 changes: 54 additions & 0 deletions src/lib/onboard/resume-provider-shim.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Wires `ensureResumeProviderReady` (in `./resume-provider-recovery`) to the
// dependencies it needs. Lives outside `src/lib/onboard.ts` so the wiring
// doesn't count against the entrypoint-budget gate.

import { DEFAULT_ROUTE_CREDENTIAL_ENV } from "../inference/config";
import { hydrateCredentialEnv } from "./credential-env";
import { validateNvidiaApiKeyValue } from "../validation";
import { D, R } from "../cli/terminal-style";
import {
ensureResumeProviderReady as ensureResumeProviderReadyImpl,
type ResumeProviderRecoveryDeps,
} from "./resume-provider-recovery";

const onboardProviders = require("./providers") as {
REMOTE_PROVIDER_CONFIG: ResumeProviderRecoveryDeps["remoteProviderConfig"];
getProviderLabel: ResumeProviderRecoveryDeps["getProviderLabel"];
};

// Lazy require breaks the circular module load — by the time
// `ensureResumeProviderReady` is called, onboard.ts has finished loading
// and its `module.exports.resumeProviderShimDeps` is populated.
type OnboardLazy = {
isNonInteractive: ResumeProviderRecoveryDeps["isNonInteractive"];
providerExistsInGateway: ResumeProviderRecoveryDeps["providerExistsInGateway"];
resumeProviderShimDeps: {
isRoutedInferenceProvider: ResumeProviderRecoveryDeps["isRoutedInferenceProvider"];
replaceNamedCredential: ResumeProviderRecoveryDeps["replaceNamedCredential"];
};
};

export async function ensureResumeProviderReady(
provider: string | null | undefined,
credentialEnv: string | null | undefined,
): Promise<{ forceInferenceSetup: boolean }> {
const o = require("../onboard") as OnboardLazy;
return ensureResumeProviderReadyImpl(provider, credentialEnv, {
remoteProviderConfig: onboardProviders.REMOTE_PROVIDER_CONFIG,
defaultRouteCredentialEnv: DEFAULT_ROUTE_CREDENTIAL_ENV,
isRoutedInferenceProvider: o.resumeProviderShimDeps.isRoutedInferenceProvider,
providerExistsInGateway: o.providerExistsInGateway,
hydrateCredentialEnv,
getProviderLabel: onboardProviders.getProviderLabel,
isNonInteractive: o.isNonInteractive,
note: (m) => console.log(`${D}${m}${R}`),
replaceNamedCredential: o.resumeProviderShimDeps.replaceNamedCredential,
validateNvidiaApiKeyValue,
log: (m) => console.log(m),
warn: (m) => console.error(m),
exit: (c) => process.exit(c),
});
}
Loading