Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4416,7 +4416,6 @@ async function runOnboard(opts: OnboardOptions = {}): Promise<void> {
console.error(" Start a fresh onboard with --name <sandbox> to choose a different name.");
process.exit(1);
}

const coreFlowContext: InitialOnboardFlowContext = {
...initialContext,
session,
Expand All @@ -4442,6 +4441,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise<void> {
providerDeps: {
checkGatewayRouteCompatibility,
preflightGatewayRouteDiscovery,
getSandboxRecoveryAuthority: providerRecovery.getSandboxRecoveryAuthority,
withGatewayRouteMutationLock: gatewayRouteMutationLock.withGatewayRouteMutationLock,
normalizeHermesAuthMethod,
setupNim: (g, s, a, recover, gateway, assertRouteCompatible, canProbeRoute) =>
Expand Down
1 change: 1 addition & 0 deletions src/lib/onboard/machine/core-flow-phases.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ function createPhases(
requiredEndpointUrl: null,
requiredInferenceApi: null,
}),
getSandboxRecoveryAuthority: (): "missing" => "missing",
withGatewayRouteMutationLock: async <T>(
_gatewayName: string,
operation: () => Promise<T> | T,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";

import { createSession } from "../../../state/onboard-session";
import type { SandboxEntry } from "../../../state/registry";
import { classifySandboxRecoveryAuthority } from "../../provider-recovery";
import { handleProviderInferenceState } from "./provider-inference";
import { baseOptions, createDeps } from "./provider-inference.test-support";

describe("provider inference recovery gating", () => {
it.each([
{ label: "fresh provider selection", fresh: true, sandboxName: "dcode-station" },
{ label: "brand-new sandbox identity (#6630)", fresh: false, sandboxName: "dc-after" },
])("disables recorded provider recovery for $label", async ({ fresh, sandboxName }) => {
const { deps, calls } = createDeps();

await handleProviderInferenceState({
...baseOptions(deps),
fresh,
sandboxName,
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
sandboxName,
null,
false,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
});

it("rejects a matching session whose sandbox step is incomplete (#6630)", async () => {
const session = createSession();
session.sandboxName = "dc-after";
const { deps, calls } = createDeps();

await handleProviderInferenceState({
...baseOptions(deps, session),
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
false,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
});

it("allows recovery for a registered sandbox", async () => {
const getAuthority = vi.fn((name: string) =>
name === "dc-after" ? ("authorized" as const) : ("missing" as const),
);
const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority });

await handleProviderInferenceState({
...baseOptions(deps),
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
true,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
expect(getAuthority).toHaveBeenCalledWith("dc-after", expect.any(String));
});

it.each([
{ label: "orphaned", reservationSessionId: undefined },
{ label: "owned by another session", reservationSessionId: "session-other" },
])("rejects an $label pending reservation despite a completed session (#6630)", async ({
reservationSessionId,
}) => {
const session = createSession();
session.sandboxName = "dc-after";
session.steps.sandbox.status = "complete";
const entry: SandboxEntry = {
name: "dc-after",
pendingRouteReservation: true,
...(reservationSessionId ? { reservationSessionId } : {}),
};
const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) =>
classifySandboxRecoveryAuthority(entry, sessionId),
);
const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority });

await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
false,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId);
});

it("allows recovery for the current session's pending reservation (#6630)", async () => {
const session = createSession();
session.sandboxName = "dc-after";
session.steps.sandbox.status = "complete";
const entry: SandboxEntry = {
name: "dc-after",
pendingRouteReservation: true,
reservationSessionId: session.sessionId,
};
const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) =>
classifySandboxRecoveryAuthority(entry, sessionId),
);
const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority });

await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
true,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId);
});

it("composes persisted route reservation ownership with resume recovery (#6626, #6630)", async () => {
const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recovery-reservation-"));
vi.stubEnv("HOME", home);
vi.resetModules();
try {
const registry = await import("../../../state/registry");
const recovery = await import("../../provider-recovery");
const sessionId = "session-current";
const route = {
provider: "compatible-endpoint",
model: "model-a",
endpointUrl: "https://api.example.test/v1",
credentialEnv: "CUSTOM_API_KEY",
preferredInferenceApi: "openai-responses",
gatewayName: "nemoclaw",
};

for (const { sandboxName, reservationSessionId, expectedRecovery } of [
{
sandboxName: "owned-reservation",
reservationSessionId: sessionId,
expectedRecovery: true,
},
{
sandboxName: "foreign-reservation",
reservationSessionId: "session-other",
expectedRecovery: false,
},
{
sandboxName: "ownerless-reservation",
reservationSessionId: undefined,
expectedRecovery: false,
},
]) {
registry.reserveSandboxInferenceRoute(sandboxName, {
...route,
reservationSessionId,
});
expect(registry.getSandbox(sandboxName)).toMatchObject({
pendingRouteReservation: true,
...(reservationSessionId ? { reservationSessionId } : {}),
});

const session = createSession({ sessionId });
session.sandboxName = sandboxName;
session.steps.sandbox.status = "complete";
const { deps, calls } = createDeps({
getSandboxRecoveryAuthority: recovery.getSandboxRecoveryAuthority,
});

await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
sandboxName,
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
sandboxName,
null,
expectedRecovery,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
}
} finally {
vi.unstubAllEnvs();
vi.resetModules();
await fs.rm(home, { recursive: true, force: true });
}
});

it("allows recovery for a matching completed session sandbox", async () => {
const session = createSession();
session.sandboxName = "dc-after";
session.steps.sandbox.status = "complete";
const { deps, calls } = createDeps();

await handleProviderInferenceState({
...baseOptions(deps, session),
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
true,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ function createDeps() {
const deps: Options["deps"] = {
checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility,
preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery,
getSandboxRecoveryAuthority: (): "missing" => "missing",
withGatewayRouteMutationLock: async (_gatewayName, operation) => await operation(),
normalizeHermesAuthMethod: () => null,
setupNim: calls.setupNim,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ export function createDeps(
deps: {
checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility,
preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery,
getSandboxRecoveryAuthority: (): "missing" => "missing",
withGatewayRouteMutationLock: async <T>(
_gatewayName: string,
operation: () => Promise<T> | T,
Expand Down
40 changes: 20 additions & 20 deletions src/lib/onboard/machine/handlers/provider-inference.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -303,26 +303,6 @@ describe("handleProviderInferenceState", () => {
});
});

it("disables recorded provider recovery during fresh provider selection", async () => {
const { deps, calls } = createDeps();

await handleProviderInferenceState({
...baseOptions(deps),
fresh: true,
sandboxName: "dcode-station",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dcode-station",
null,
false,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
});

it("does not use resume shortcuts when fresh is also set", async () => {
const session = createSession({ provider: "ollama-local", model: "llama3.1" });
session.steps.provider_selection.status = "complete";
Expand Down Expand Up @@ -1139,6 +1119,26 @@ describe("handleProviderInferenceState", () => {
const result = await handleProviderInferenceState(baseOptions(deps));

expect(setupNim).toHaveBeenCalledTimes(2);
expect(setupNim).toHaveBeenNthCalledWith(
1,
{ type: "nvidia" },
null,
null,
true,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
expect(setupNim).toHaveBeenNthCalledWith(
2,
{ type: "nvidia" },
"my-assistant",
null,
false,
"nemoclaw",
expect.any(Function),
expect.any(Function),
);
expect(setupInference).toHaveBeenCalledTimes(2);
expect(result.model).toBe("good");
expect(calls.startStep).toHaveBeenCalledWith("provider_selection");
Expand Down
19 changes: 18 additions & 1 deletion src/lib/onboard/machine/handlers/provider-inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ import type {
} from "../../../inference/gateway-route-compatibility";
import type { WebSearchConfig } from "../../../inference/web-search";
import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session";
import {
type SandboxRecoveryAuthority,
shouldRecoverRecordedProvider,
} from "../../provider-recovery";
import { withInferenceTrace, withProviderSelectionTrace } from "../../tracing";
import { advanceTo, type OnboardStateTransitionResult, retryTo } from "../result";
import {
Expand Down Expand Up @@ -89,6 +93,10 @@ export interface ProviderInferenceStateOptions<Gpu, Agent, Host> {
deps: {
checkGatewayRouteCompatibility: CurrentGatewayRouteCompatibilityCheck;
preflightGatewayRouteDiscovery: CurrentGatewayRouteDiscoveryPreflight;
getSandboxRecoveryAuthority(
sandboxName: string,
sessionId: string | null | undefined,
): SandboxRecoveryAuthority;
withGatewayRouteMutationLock<T>(
gatewayName: string,
operation: () => Promise<T> | T,
Expand Down Expand Up @@ -442,6 +450,15 @@ export async function handleProviderInferenceState<Gpu, Agent, Host>({
}
} else {
await deps.startRecordedStep("provider_selection");
const recoverRecordedProvider = shouldRecoverRecordedProvider({
fresh,
sandboxName,
sandboxRecoveryAuthority: sandboxName
? deps.getSandboxRecoveryAuthority(sandboxName, session?.sessionId)
: "missing",
sessionSandboxName:
session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null,
});
const selection = await withProviderSelectionTrace(
sandboxName,
(agent as { name?: string } | null)?.name,
Expand All @@ -450,7 +467,7 @@ export async function handleProviderInferenceState<Gpu, Agent, Host>({
gpu,
sandboxName,
agent,
!fresh,
recoverRecordedProvider,
gatewayName,
(route) => guardProviderInferenceRouteSelection(deps, gatewayName, sandboxName, route),
(provider) =>
Expand Down
Loading
Loading