Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions src/lib/adapters/docker/pull.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,65 @@ describe("docker pull progress watchdog", () => {
});
});

it("allows quiet pull finalization within the 15 minute stall timeout", async () => {
vi.useFakeTimers();
const child = new FakeChild();
const pull = dockerPullWithProgressWatchdog("example/image:latest", {
suppressOutput: true,
spawnImpl: () => child,
});

child.stderr.emit("data", Buffer.from("abc123def: Pull complete\n"));
await vi.advanceTimersByTimeAsync(15 * 60_000 - 10_000);
expect(child.kill).not.toHaveBeenCalled();

child.emit("close", 0, null);
await expect(pull).resolves.toMatchObject({
status: 0,
timedOut: false,
timeoutKind: null,
});
});

it("kills a quiet pull after the default 15 minute stall timeout", async () => {
vi.useFakeTimers();
const child = new FakeChild();
const pull = dockerPullWithProgressWatchdog("example/image:latest", {
suppressOutput: true,
spawnImpl: () => child,
});

child.stderr.emit("data", Buffer.from("abc123def: Pull complete\n"));
await vi.advanceTimersByTimeAsync(16 * 60_000);

await expect(pull).resolves.toMatchObject({
status: 124,
timedOut: true,
timeoutKind: "stall",
});
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
});

it("respects a custom stall timeout", async () => {
vi.useFakeTimers();
const child = new FakeChild();
const pull = dockerPullWithProgressWatchdog("example/image:latest", {
suppressOutput: true,
stallTimeoutMs: 5 * 60_000,
spawnImpl: () => child,
});

child.stderr.emit("data", Buffer.from("abc123def: Pull complete\n"));
await vi.advanceTimersByTimeAsync(5 * 60_000);

await expect(pull).resolves.toMatchObject({
status: 124,
timedOut: true,
timeoutKind: "stall",
});
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
});

it("kills a pull when output repeats without forward progress", async () => {
vi.useFakeTimers();
const child = new FakeChild();
Expand Down
6 changes: 5 additions & 1 deletion src/lib/adapters/docker/pull.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ export function dockerPull(imageRef: string, opts: DockerRunOptions = {}): Docke
return dockerRun(["pull", imageRef], opts);
}

export const DEFAULT_DOCKER_PULL_STALL_TIMEOUT_MS = 120 * 1000;
// DGX Spark vLLM pulls can spend several minutes quiet while Docker finalizes
// large NGC layers. Keep the stall watchdog above the reported ~5.5 minute
// successful control pull, while the separate max timeout still bounds total
// wall-clock runtime.
const DEFAULT_DOCKER_PULL_STALL_TIMEOUT_MS = 15 * 60 * 1000;
export const DEFAULT_DOCKER_PULL_MAX_TIMEOUT_MS = 12 * 60 * 60 * 1000;
const DOCKER_PULL_OUTPUT_TAIL_LINES = 200;
const DOCKER_PULL_PROGRESS_STATE_LIMIT = 512;
Expand Down
3 changes: 3 additions & 0 deletions src/lib/inference/vllm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,9 @@ function dockerPrereqsOk(): { ok: boolean; reason?: string } {

export async function pullImage(profile: VllmProfile): Promise<{ ok: boolean; reason?: string }> {
emit(`Pulling vLLM image: ${profile.image}`);
// Docker can be quiet while finalizing large layers on every supported vLLM
// profile, so all profiles intentionally share the 15-minute stall default.
// The profile-specific maximum still bounds the complete pull operation.
const result = await dockerPullWithProgressWatchdog(profile.image, {
maxTimeoutMs: profile.pullTimeoutSec * 1000,
logLine: emit,
Expand Down