diff --git a/src/lib/adapters/docker/pull.test.ts b/src/lib/adapters/docker/pull.test.ts index 5b70562d712..aa6066a944e 100644 --- a/src/lib/adapters/docker/pull.test.ts +++ b/src/lib/adapters/docker/pull.test.ts @@ -107,6 +107,65 @@ describe("docker pull progress watchdog", () => { }); }); + it("allows quiet pull finalization within the 15 minute stall timeout", async () => { + vi.useFakeTimers(); + const child = new FakeChild(); + const pull = dockerPullWithProgressWatchdog("example/image:latest", { + suppressOutput: true, + spawnImpl: () => child, + }); + + child.stderr.emit("data", Buffer.from("abc123def: Pull complete\n")); + await vi.advanceTimersByTimeAsync(15 * 60_000 - 10_000); + expect(child.kill).not.toHaveBeenCalled(); + + child.emit("close", 0, null); + await expect(pull).resolves.toMatchObject({ + status: 0, + timedOut: false, + timeoutKind: null, + }); + }); + + it("kills a quiet pull after the default 15 minute stall timeout", async () => { + vi.useFakeTimers(); + const child = new FakeChild(); + const pull = dockerPullWithProgressWatchdog("example/image:latest", { + suppressOutput: true, + spawnImpl: () => child, + }); + + child.stderr.emit("data", Buffer.from("abc123def: Pull complete\n")); + await vi.advanceTimersByTimeAsync(16 * 60_000); + + await expect(pull).resolves.toMatchObject({ + status: 124, + timedOut: true, + timeoutKind: "stall", + }); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + }); + + it("respects a custom stall timeout", async () => { + vi.useFakeTimers(); + const child = new FakeChild(); + const pull = dockerPullWithProgressWatchdog("example/image:latest", { + suppressOutput: true, + stallTimeoutMs: 5 * 60_000, + spawnImpl: () => child, + }); + + child.stderr.emit("data", Buffer.from("abc123def: Pull complete\n")); + await vi.advanceTimersByTimeAsync(5 * 60_000); + + await expect(pull).resolves.toMatchObject({ + status: 124, + timedOut: true, + timeoutKind: "stall", + }); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + }); + it("kills a pull when output repeats without forward progress", async () => { vi.useFakeTimers(); const child = new FakeChild(); diff --git a/src/lib/adapters/docker/pull.ts b/src/lib/adapters/docker/pull.ts index 7f2170fbe72..8daa1534938 100644 --- a/src/lib/adapters/docker/pull.ts +++ b/src/lib/adapters/docker/pull.ts @@ -12,7 +12,11 @@ export function dockerPull(imageRef: string, opts: DockerRunOptions = {}): Docke return dockerRun(["pull", imageRef], opts); } -export const DEFAULT_DOCKER_PULL_STALL_TIMEOUT_MS = 120 * 1000; +// DGX Spark vLLM pulls can spend several minutes quiet while Docker finalizes +// large NGC layers. Keep the stall watchdog above the reported ~5.5 minute +// successful control pull, while the separate max timeout still bounds total +// wall-clock runtime. +const DEFAULT_DOCKER_PULL_STALL_TIMEOUT_MS = 15 * 60 * 1000; export const DEFAULT_DOCKER_PULL_MAX_TIMEOUT_MS = 12 * 60 * 60 * 1000; const DOCKER_PULL_OUTPUT_TAIL_LINES = 200; const DOCKER_PULL_PROGRESS_STATE_LIMIT = 512; diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index f06b4cb8dbd..11f4ea18eff 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -227,6 +227,9 @@ function dockerPrereqsOk(): { ok: boolean; reason?: string } { export async function pullImage(profile: VllmProfile): Promise<{ ok: boolean; reason?: string }> { emit(`Pulling vLLM image: ${profile.image}`); + // Docker can be quiet while finalizing large layers on every supported vLLM + // profile, so all profiles intentionally share the 15-minute stall default. + // The profile-specific maximum still bounds the complete pull operation. const result = await dockerPullWithProgressWatchdog(profile.image, { maxTimeoutMs: profile.pullTimeoutSec * 1000, logLine: emit,