Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 9 additions & 13 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -555,8 +555,8 @@ import {
type SandboxGpuConfig,
type SandboxGpuFlag,
} from "./onboard/sandbox-gpu-mode";
import { filterSlackSelectionByValidation } from "./onboard/slack-validation";
import type { SelectionDrift } from "./onboard/selection-drift";
import { filterSlackSelectionByValidation } from "./onboard/slack-validation";
import { formatOnboardConfigSummary, formatSandboxBuildEstimateNote } from "./onboard/summary";
import type { ModelValidationResult, ValidationFailureLike } from "./onboard/types";
import type { ContainerRuntime } from "./platform";
Expand Down Expand Up @@ -3682,26 +3682,25 @@ async function createSandbox(
// without this gate, NemoClaw registers a phantom sandbox that
// causes "sandbox not found" on every subsequent connect/status call.
console.log(" Waiting for sandbox to become ready...");
const ready = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({
const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({
sandboxName,
timeoutSecs: sandboxReadyTimeoutSecs,
runCaptureOpenshell,
isSandboxReady,
getSandboxFailurePhase: gatewayState.getSandboxFailurePhase,
sleep: sleepSeconds,
});

const restoreBackupPath =
pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath;

if (!ready) {
if (!readiness.ready) {
const diagnostics = sandboxCreateFailureDiagnostics.collectSandboxCreateFailureDiagnostics(
sandboxName,
{ backupPath: restoreBackupPath },
);
console.error("");
console.error(
` Sandbox '${sandboxName}' was created but did not become ready within ${sandboxReadyTimeoutSecs}s.`,
);
sandboxReadinessTracing.printReadinessFailure(readiness, sandboxName, sandboxReadyTimeoutSecs);
if (diagnostics) {
console.error(` Diagnostics saved: ${diagnostics.dir}`);
if (diagnostics.summaryLines.length > 0) {
Expand All @@ -3715,11 +3714,7 @@ async function createSandbox(
}
}
if (useDockerGpuPatch) {
dockerGpuPatch.printDockerGpuReadinessFailure(
sandboxName,
dockerGpuCreatePatch.selectedMode(),
{ runCaptureOpenshell },
);
dockerGpuCreatePatch.printReadinessFailureIfEnabled();
} else {
// Clean up non-GPU failures after preserving local diagnostics so the
// next onboard retry with the same name does not fail on "sandbox already exists".
Expand Down Expand Up @@ -3749,13 +3744,14 @@ async function createSandbox(
});

if (effectiveSandboxGpuConfig.sandboxGpuEnabled) {
// Runs the GPU proof, then (when the Docker GPU patch recreated the
// container) gates on host-network local inference reachability (#4509).
// Runs the GPU proof, preserving Docker-GPU patch Error-phase diagnostics
// when applicable, then gates host-network local inference reachability (#4509).
dockerGpuLocalInference.verifyGpuSandboxAfterReady(effectiveSandboxGpuConfig, provider, {
sandboxName,
dockerDriverGateway: isLinuxDockerDriverGatewayEnabled(),
useDockerGpuPatch,
verifyDirectSandboxGpu,
verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit,
selectedMode: dockerGpuCreatePatch.selectedMode,
runCaptureOpenshell,
log: console.log,
Expand Down
37 changes: 37 additions & 0 deletions src/lib/onboard/docker-gpu-local-inference.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,43 @@ describe("verifyGpuSandboxAfterReady", () => {
expect(log).toHaveBeenCalledWith(expect.stringContaining("reachable from sandbox"));
});

it("uses Docker GPU patch verifier when supplied", () => {
const verifyDirectSandboxGpu = vi.fn();
const verifyGpuOrExit = vi.fn((proof: (sandboxName: string) => void) => proof("alpha"));
verifyGpuSandboxAfterReady(
GPU_CONFIG,
"vllm-local",
baseOptions({
verifyDirectSandboxGpu,
verifyGpuOrExit,
deps: {
findContainerIds: () => ["container-abc"],
dockerCapture: vi.fn(() => inspectWithNetworkMode("host")),
dockerRun: dockerRunWithCurl({ status: 0 }),
sleep: vi.fn(),
},
}),
);
expect(verifyGpuOrExit).toHaveBeenCalledWith(verifyDirectSandboxGpu);
expect(verifyDirectSandboxGpu).toHaveBeenCalledWith("alpha");
});

it("does not duplicate proof diagnostics when Docker GPU patch verifier handles them", () => {
const proofError = new Error("process.exit");
const verifyGpuOrExit = vi.fn(() => {
throw proofError;
});
const logError = vi.fn();
expect(() =>
verifyGpuSandboxAfterReady(
GPU_CONFIG,
"ollama-local",
baseOptions({ verifyGpuOrExit, logError }),
),
).toThrow(proofError);
expect(logError).not.toHaveBeenCalled();
});

it("routes failure diagnostics through the provided error sink and exits", () => {
const logError = vi.fn();
const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => {
Expand Down
18 changes: 14 additions & 4 deletions src/lib/onboard/docker-gpu-local-inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ export type GpuSandboxAfterReadyOptions = {
dockerDriverGateway: boolean;
useDockerGpuPatch: boolean;
verifyDirectSandboxGpu: (sandboxName: string) => void;
verifyGpuOrExit?: (verifyDirectSandboxGpu: (sandboxName: string) => void) => void;
selectedMode: () => DockerGpuPatchMode | null;
runCaptureOpenshell: (args: string[], opts?: Record<string, unknown>) => string;
env?: NodeJS.ProcessEnv;
Expand All @@ -392,11 +393,20 @@ export function verifyGpuSandboxAfterReady(
options: GpuSandboxAfterReadyOptions,
): void {
try {
options.verifyDirectSandboxGpu(options.sandboxName);
if (options.verifyGpuOrExit) {
options.verifyGpuOrExit(options.verifyDirectSandboxGpu);
} else {
options.verifyDirectSandboxGpu(options.sandboxName);
}
} catch (error) {
printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), {
runCaptureOpenshell: options.runCaptureOpenshell,
});
// `verifyGpuOrExit` is supplied by the Docker GPU create patch and already
// prints the richer Error-phase / patched-container diagnostics before
// rethrowing. Avoid a second generic proof-failure block in that path.
if (!options.verifyGpuOrExit) {
printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), {
runCaptureOpenshell: options.runCaptureOpenshell,
});
}
throw error;
}

Expand Down
Loading
Loading