Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 53 additions & 20 deletions src/lib/actions/inference-set.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,24 @@
} as Session;
}

function expectInferenceRegistryUpdate(
Comment thread
github-code-quality[bot] marked this conversation as resolved.
Fixed
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
updateSandbox: ReturnType<typeof vi.fn>,
sandboxName: string,
expected: { provider: string; model: string; preferredInferenceApi?: string | null },
): void {
expect(updateSandbox).toHaveBeenCalledWith(
sandboxName,
expect.objectContaining({
provider: expected.provider,
model: expected.model,
endpointUrl: null,
credentialEnv: null,
preferredInferenceApi: expected.preferredInferenceApi ?? null,
nimContainer: null,
}),
);
}

function createDeps(options: {
config: ConfigObject;
entry?: SandboxEntry | null;
Expand Down Expand Up @@ -453,10 +471,13 @@
});
expect(deps.calls.writeSandboxConfig).toHaveBeenCalledWith("alpha", OPENCLAW_TARGET, config);
expect(deps.calls.recomputeSandboxConfigHash).toHaveBeenCalledWith("alpha", OPENCLAW_TARGET);
expect(deps.calls.updateSandbox).toHaveBeenCalledWith("alpha", {
provider: "nvidia-prod",
model: "nvidia/nemotron-3-super-120b-a12b",
});
expect(deps.calls.updateSandbox).toHaveBeenCalledWith(
"alpha",
expect.objectContaining({
provider: "nvidia-prod",
model: "nvidia/nemotron-3-super-120b-a12b",
}),
);
expect(deps.getSession()).toMatchObject({
provider: "nvidia-prod",
model: "nvidia/nemotron-3-super-120b-a12b",
Expand Down Expand Up @@ -545,10 +566,13 @@
"/sandbox/.hermes/config.yaml",
);
expect(deps.calls.recomputeSandboxConfigHash).toHaveBeenCalledWith("hermes", HERMES_TARGET);
expect(deps.calls.updateSandbox).toHaveBeenCalledWith("hermes", {
provider: "hermes-provider",
model: "openai/gpt-5.4-mini",
});
expect(deps.calls.updateSandbox).toHaveBeenCalledWith(
"hermes",
expect.objectContaining({
provider: "hermes-provider",
model: "openai/gpt-5.4-mini",
}),
);
expect(deps.getSession()).toMatchObject({
provider: "hermes-provider",
model: "openai/gpt-5.4-mini",
Expand Down Expand Up @@ -823,10 +847,13 @@
await runInferenceSet({ provider: "hermes-provider", model: "z-ai/glm-5.1" }, deps);

expect(deps.calls.writeSandboxConfig).toHaveBeenCalledWith("hermes-one", HERMES_TARGET, config);
expect(deps.calls.updateSandbox).toHaveBeenCalledWith("hermes-one", {
provider: "hermes-provider",
model: "z-ai/glm-5.1",
});
expect(deps.calls.updateSandbox).toHaveBeenCalledWith(
"hermes-one",
expect.objectContaining({
provider: "hermes-provider",
model: "z-ai/glm-5.1",
}),
);
});

it("requires --sandbox when the nemohermes alias cannot choose one Hermes sandbox", async () => {
Expand Down Expand Up @@ -899,10 +926,13 @@
);

// Registry still updated despite the in-sandbox sync throwing (no stale registry → no revert).
expect(deps.calls.updateSandbox).toHaveBeenCalledWith("alpha", {
provider: "nvidia-prod",
model: "nvidia/nemotron-3-super-120b-a12b",
});
expect(deps.calls.updateSandbox).toHaveBeenCalledWith(
"alpha",
expect.objectContaining({
provider: "nvidia-prod",
model: "nvidia/nemotron-3-super-120b-a12b",
}),
);
expect(deps.calls.recomputeSandboxConfigHash).not.toHaveBeenCalled();
expect(result).toMatchObject({
provider: "nvidia-prod",
Expand Down Expand Up @@ -940,10 +970,13 @@

// Config write happened and registry is updated; the run resolves without aborting.
expect(deps.calls.writeSandboxConfig).toHaveBeenCalled();
expect(deps.calls.updateSandbox).toHaveBeenCalledWith("alpha", {
provider: "nvidia-prod",
model: "nvidia/nemotron-3-super-120b-a12b",
});
expect(deps.calls.updateSandbox).toHaveBeenCalledWith(
"alpha",
expect.objectContaining({
provider: "nvidia-prod",
model: "nvidia/nemotron-3-super-120b-a12b",
}),
);
expect(result).toMatchObject({ inSandboxConfigSynced: false });

// Degraded: warns about the stale integrity hash, points at rebuild, no "synced".
Expand Down
25 changes: 19 additions & 6 deletions src/lib/actions/inference-set.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
type SandboxInferenceConfig,
} from "../inference/config";
import { resolveContextWindowForModel } from "../inference/context-window";
import { inferenceSelectionRegistryFields } from "../inference/selection";
import { type ValidationResult, validateLocalProvider } from "../inference/local";
import { ensureLocalProviderReachable } from "../onboard/local-inference-topology";
import {
Expand Down Expand Up @@ -429,12 +430,6 @@ export async function runInferenceSet(
);
}

// Write the registry before the crash-prone in-sandbox sync so the gateway
// and registry can't end up split (#3725) and trigger a revert on connect (#3726).
if (!deps.updateSandbox(sandboxName, { provider, model })) {
throw new InferenceSetError(`Failed to update NemoClaw registry for sandbox '${sandboxName}'.`);
}

const config = deps.readSandboxConfig(sandboxName, target);
const preferredInferenceApi = resolveRuntimeInferenceApi({
agentName,
Expand All @@ -444,6 +439,24 @@ export async function runInferenceSet(
sandboxName,
session: deps.loadSession(),
});
// Write the registry before the crash-prone in-sandbox sync so the gateway
// and registry can't end up split (#3725) and trigger a revert on connect (#3726).
if (
!deps.updateSandbox(
sandboxName,
inferenceSelectionRegistryFields({
provider,
model,
endpointUrl: entry.provider === provider ? (entry.endpointUrl ?? null) : null,
credentialEnv: entry.credentialEnv ?? null,
preferredInferenceApi,
nimContainer: entry.nimContainer ?? null,
}),
)
) {
throw new InferenceSetError(`Failed to update NemoClaw registry for sandbox '${sandboxName}'.`);
}

Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
let patched: { changed: boolean; route: SandboxInferenceConfig };
if (agentName === "hermes") {
patched = patchHermesInferenceConfig(config, provider, model, preferredInferenceApi);
Expand Down
38 changes: 36 additions & 2 deletions src/lib/actions/sandbox/rebuild-resume-config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,18 @@ describe("getRebuildEndpointFromRegistry", () => {
expect(result.endpointUrl.length).toBeGreaterThan(0);
});

it("marks a custom OpenAI-compatible provider as unknown (session-only URL)", () => {
it("marks a custom OpenAI-compatible provider as unknown without durable endpoint metadata", () => {
expect(getRebuildEndpointFromRegistry("compatible-endpoint")).toEqual({ known: false });
});

it("uses durable custom endpoint metadata from the sandbox registry", () => {
expect(
getRebuildEndpointFromRegistry("compatible-endpoint", "http://127.0.0.1:19999/v1"),
).toEqual({
known: true,
endpointUrl: "http://127.0.0.1:19999/v1",
});
});
});

describe("prepareRebuildResumeConfig", () => {
Expand Down Expand Up @@ -102,7 +111,7 @@ describe("prepareRebuildResumeConfig", () => {
expect(typeof config?.endpointUrl).toBe("string");
});

it("fails closed for a custom endpoint with a non-matching session", () => {
it("fails closed for a custom endpoint with a non-matching session and no registry endpoint", () => {
vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "other" });
expect(() =>
prepareRebuildResumeConfig(
Expand All @@ -115,6 +124,31 @@ describe("prepareRebuildResumeConfig", () => {
).toThrow("Cannot determine recreate endpoint");
});

it("recreates custom endpoints from durable registry metadata when the session is unrelated", () => {
vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "other" });
const config = prepareRebuildResumeConfig(
"alpha",
entry({
provider: "compatible-endpoint",
model: "m",
endpointUrl: "http://127.0.0.1:19999/v1",
credentialEnv: "COMPATIBLE_API_KEY",
preferredInferenceApi: "openai-completions",
}),
null,
noopLog,
throwingBail,
);
expect(config).toMatchObject({
provider: "compatible-endpoint",
model: "m",
credentialEnv: "COMPATIBLE_API_KEY",
preferredInferenceApi: "openai-completions",
pinEndpoint: true,
endpointUrl: "http://127.0.0.1:19999/v1",
});
});

it("surfaces an ambient agent mismatch in the assessment", () => {
vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha" });
const prior = process.env.NEMOCLAW_AGENT;
Expand Down
62 changes: 41 additions & 21 deletions src/lib/actions/sandbox/rebuild-resume-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import { CLI_NAME } from "../../cli/branding";
import { RD as _RD, D, R } from "../../cli/terminal-style";
import { normalizeInferenceSelection } from "../../inference/selection";
import * as onboardSession from "../../state/onboard-session";
import {
type AmbientRecreateEnvAssessment,
Expand Down Expand Up @@ -39,7 +40,9 @@ export function isLocalInferenceProvider(provider: string | null | undefined): p
/** Resolve the credential environment variable required to recreate a sandbox. */
export function getRebuildCredentialEnvFromRegistry(
provider: string | null | undefined,
recordedCredentialEnv?: string | null,
): string | null {
if (recordedCredentialEnv) return recordedCredentialEnv;
if (!provider || isLocalInferenceProvider(provider)) {
return null;
}
Expand Down Expand Up @@ -71,19 +74,26 @@ const SESSION_ONLY_ENDPOINT_PROVIDER_NAMES = new Set(
* recreate endpoint can be re-derived without the target's own onboard session —
* a known remote provider with a canonical URL (e.g. nvidia-prod → NVIDIA
* Endpoints), a local or routed (blueprint-derived) provider (no static URL to
* pin), or any other provider that does not record a custom base URL. Returns
* `{ known: false }` only for custom OpenAI/Anthropic-compatible providers whose
* base URL lives solely in their own session — the caller must then refuse to
* destroy the sandbox from an unrelated session rather than guess the endpoint.
* pin), or a custom OpenAI/Anthropic-compatible provider with durable registry
* metadata. Returns `{ known: false }` only for custom providers whose base URL
* is absent from both the selected sandbox registry entry and its own session —
* the caller must then refuse to destroy the sandbox from an unrelated session
* rather than guess the endpoint.
*/
export function getRebuildEndpointFromRegistry(
provider: string | null | undefined,
recordedEndpointUrl?: string | null,
): { known: true; endpointUrl: string | null } | { known: false } {
if (!provider) return { known: true, endpointUrl: null };
if (isLocalInferenceProvider(provider)) return { known: true, endpointUrl: null };
// Custom OpenAI/Anthropic-compatible providers carry their base URL only in
// the session; without a matching session it cannot be recovered.
if (SESSION_ONLY_ENDPOINT_PROVIDER_NAMES.has(provider)) return { known: false };
// the selected sandbox's durable metadata or its own onboard session; never
// borrow the base URL from an unrelated session.
if (SESSION_ONLY_ENDPOINT_PROVIDER_NAMES.has(provider)) {
return recordedEndpointUrl
? { known: true, endpointUrl: recordedEndpointUrl }
: { known: false };
}
const remoteConfig =
provider === "nvidia-nim"
? REMOTE_PROVIDER_CONFIG.build
Expand All @@ -106,6 +116,7 @@ export interface RebuildResumeConfig {
readonly model: string | null;
readonly nimContainer: string | null;
readonly credentialEnv: string | null;
readonly preferredInferenceApi: string | null;
/** Overwrite the session endpoint with `endpointUrl`; false keeps a matching session's own custom URL. */
readonly pinEndpoint: boolean;
readonly endpointUrl: string | null;
Expand Down Expand Up @@ -156,26 +167,31 @@ export function prepareRebuildResumeConfig(

const session = onboardSession.loadSession();
const sessionMatchesSandbox = session?.sandboxName === sandboxName;
const rebuildEndpoint = getRebuildEndpointFromRegistry(sb.provider);
const registrySelection = normalizeInferenceSelection(sb);
const rebuildEndpoint = getRebuildEndpointFromRegistry(
registrySelection.provider,
registrySelection.endpointUrl,
);

// When the loaded session belongs to a *different* sandbox (e.g. an
// installer's just-completed onboard before `upgrade-sandboxes --auto`), the
// target's inference endpoint can only be re-derived for providers with a
// canonical endpoint (NVIDIA Endpoints, Anthropic, etc.), local inference, or
// routed inference. For a custom OpenAI-compatible provider the base URL lives
// only in the target's own session — which we don't have — so recreating would
// either fail or silently reconfigure against the unrelated session's
// endpoint. Fail closed before any destructive work so the sandbox stays live.
// canonical endpoint (NVIDIA Endpoints, Anthropic, etc.), local inference,
// routed inference, or durable custom endpoint metadata recorded on the target
// registry entry. For legacy custom OpenAI-compatible entries without that
// metadata, recreating would either fail or silently reconfigure against the
// unrelated session's endpoint. Fail closed before any destructive work so the
// sandbox stays live.
if (
!sessionMatchesSandbox &&
sb.provider &&
!isLocalInferenceProvider(sb.provider) &&
sb.provider !== hermesProviderAuth.HERMES_PROVIDER_NAME &&
registrySelection.provider &&
!isLocalInferenceProvider(registrySelection.provider) &&
registrySelection.provider !== hermesProviderAuth.HERMES_PROVIDER_NAME &&
!rebuildEndpoint.known
) {
console.error("");
console.error(
` ${_RD}Rebuild preflight failed:${R} cannot determine the inference endpoint for provider '${sb.provider}'.`,
` ${_RD}Rebuild preflight failed:${R} cannot determine the inference endpoint for provider '${registrySelection.provider}'.`,
);
console.error(
` The custom endpoint for '${sandboxName}' is recorded only in its own onboard session,`,
Expand All @@ -186,17 +202,21 @@ export function prepareRebuildResumeConfig(
console.error("");
console.error(" Sandbox is untouched — no data was lost.");
bail(
`Cannot determine recreate endpoint for provider '${sb.provider}' without a matching session`,
`Cannot determine recreate endpoint for provider '${registrySelection.provider}' without a matching session`,
);
return null;
}

return {
agent: rebuildAgent,
provider: sb.provider ?? null,
model: sb.model ?? null,
nimContainer: sb.nimContainer ?? null,
credentialEnv: getRebuildCredentialEnvFromRegistry(sb.provider),
provider: registrySelection.provider,
model: registrySelection.model,
nimContainer: registrySelection.nimContainer,
credentialEnv: getRebuildCredentialEnvFromRegistry(
registrySelection.provider,
registrySelection.credentialEnv,
),
preferredInferenceApi: registrySelection.preferredInferenceApi,
pinEndpoint: !sessionMatchesSandbox && rebuildEndpoint.known,
endpointUrl: rebuildEndpoint.known ? rebuildEndpoint.endpointUrl : null,
ambient,
Expand Down
5 changes: 3 additions & 2 deletions src/lib/actions/sandbox/rebuild.ts
Original file line number Diff line number Diff line change
Expand Up @@ -426,8 +426,8 @@ function preflightRebuildCredentials(
// omitted credentialEnv; rebuild rewrites provider/model from this entry later,
// so remote registry providers must still fail closed before backup/delete.
let rebuildCredentialEnv = sessionMatchesTarget
? session?.credentialEnv || getRebuildCredentialEnvFromRegistry(sb.provider)
: getRebuildCredentialEnvFromRegistry(sb.provider);
? session?.credentialEnv || getRebuildCredentialEnvFromRegistry(sb.provider, sb.credentialEnv)
: getRebuildCredentialEnvFromRegistry(sb.provider, sb.credentialEnv);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (!sessionMatchesTarget && session?.sandboxName) {
log(
`Preflight warning: session belongs to '${session.sandboxName}', not '${sandboxName}' — using registry credential env ${rebuildCredentialEnv || "(none)"}`,
Expand Down Expand Up @@ -781,6 +781,7 @@ export async function rebuildSandbox(
s.model = resumeConfig.model;
s.nimContainer = resumeConfig.nimContainer;
s.credentialEnv = resumeConfig.credentialEnv;
s.preferredInferenceApi = resumeConfig.preferredInferenceApi;
if (resumeConfig.pinEndpoint) {
s.endpointUrl = resumeConfig.endpointUrl;
}
Expand Down
Loading
Loading