Repository navigation
fix(inference): classify llama.cpp image pull failures #10561
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
4e8ab2d
3d42cdc
c65e9bb
1d3ef40
a46bc6b
77a64ac
2634729
249e92b
ac6263c
ec3c1f9
40730bd
6494541
896909c
4bebeff
3f0feaf
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -121,6 +121,8 @@ interface DockerNetworkInspection { | |
| interface ManagedLlamaCppImagePullResult { | ||
| readonly status: number | null; | ||
| readonly error?: Error; | ||
| readonly stderr?: string; | ||
| readonly stdout?: string; | ||
| } | ||
|
|
||
| type ManagedLlamaCppImagePull = ( | ||
|
|
@@ -132,6 +134,71 @@ type ManagedLlamaCppImagePull = ( | |
| }, | ||
| ) => Promise<ManagedLlamaCppImagePullResult>; | ||
|
|
||
| type ManagedLlamaCppImagePullFailureLayer = | ||
| | "authentication" | ||
| | "daemon behavior" | ||
| | "invalid dependency" | ||
| | "registry availability" | ||
| | "runner network" | ||
| | "storage" | ||
| | "unclassified"; | ||
|
|
||
| const IMAGE_PULL_LAYER_PATTERNS: ReadonlyArray<{ | ||
| readonly layer: ManagedLlamaCppImagePullFailureLayer; | ||
| readonly pattern: RegExp; | ||
| }> = [ | ||
| { | ||
| layer: "authentication", | ||
| pattern: | ||
| /(?:authentication required|unauthorized|pull access denied|requested access .* denied|credential helper|credentials? store)/iu, | ||
| }, | ||
| { | ||
| layer: "storage", | ||
| pattern: /(?:no space left on device|disk quota exceeded|insufficient disk space)/iu, | ||
| }, | ||
| { | ||
| layer: "runner network", | ||
| pattern: | ||
| /(?:dial tcp|no such host|temporary failure in name resolution|network is unreachable|connection (?:refused|reset)|tls handshake timeout|i\/o timeout|context deadline exceeded|client\.timeout|certificate signed by unknown authority)/iu, | ||
| }, | ||
| { | ||
| layer: "invalid dependency", | ||
| pattern: | ||
| /(?:manifest unknown|manifest .* not found|no matching manifest|no match for platform)/iu, | ||
| }, | ||
| { | ||
| layer: "registry availability", | ||
| pattern: | ||
| /(?:too many requests|rate limit|service unavailable|bad gateway|gateway timeout|status(?: code)?:? 5\d\d|unexpected status.*5\d\d)/iu, | ||
| }, | ||
| { | ||
| layer: "daemon behavior", | ||
| pattern: | ||
| /(?:cannot connect to the (?:docker|container) daemon|daemon is not running|error during connect|docker pull failed to start)/iu, | ||
| }, | ||
| ]; | ||
|
|
||
| function classifyImagePullFailureLayer(diagnostic: string): ManagedLlamaCppImagePullFailureLayer { | ||
| return ( | ||
| IMAGE_PULL_LAYER_PATTERNS.find(({ pattern }) => pattern.test(diagnostic))?.layer ?? | ||
| "unclassified" | ||
| ); | ||
| } | ||
|
|
||
| function imagePullFailureDiagnostic(result: ManagedLlamaCppImagePullResult): string { | ||
| const sources = [result.stdout, result.stderr, result.error?.message].filter( | ||
| (value): value is string => typeof value === "string" && value.trim().length > 0, | ||
| ); | ||
| const layer = classifyImagePullFailureLayer(sources.join("\n")); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] Preserve daemon-error precedence. The classifier returns the first matching layer, but runner network is checked before daemon behavior. Real Docker errors commonly combine error during connect with dial tcp or connection refused, so those daemon failures are mislabeled as runner network. Move the daemon-specific rule ahead of the broad network rule, or encode explicit precedence, and add a mixed-signal regression that expects daemon behavior. |
||
| const evidence = | ||
| layer === "unclassified" | ||
| ? "No recognized failure-layer signature matched." | ||
| : "A pull-output signature matched this layer."; | ||
| const status = | ||
| result.status === null ? "Exit status unavailable." : `Exit status: ${String(result.status)}.`; | ||
| return `Failure classification: ${layer}. ${evidence} ${status} Raw pull output is not included.`; | ||
| } | ||
|
|
||
| function requireSuccess(label: string, result: ReturnType<ContainerEngine["capture"]>): string { | ||
| if (result.error || result.status !== 0) { | ||
| throw new Error(`Managed llama.cpp ${label} failed (exit ${String(result.status)}).`); | ||
|
|
@@ -358,11 +425,13 @@ async function pullExactImages( | |
| ? await pull(image, { | ||
| env: dockerEnv, | ||
| maxTimeoutMs: IMAGE_PULL_TIMEOUT_MS, | ||
| logLine: (line) => log(` ${line}`), | ||
| logLine: () => undefined, | ||
| }) | ||
| : engine.capture(["pull", image], IMAGE_PULL_TIMEOUT_MS); | ||
| if (result.status !== 0 || ("error" in result && result.error !== undefined)) { | ||
| throw new Error(`Pinned managed-inference image pull failed for ${image}.`); | ||
| throw new Error( | ||
| `Pinned managed-inference image pull failed for ${image}. ${imagePullFailureDiagnostic(result)}`, | ||
| ); | ||
| } | ||
| const after = engine.capture(["image", "inspect", image], DOCKER_INSPECT_TIMEOUT_MS); | ||
| if (after.error || after.status !== 0) { | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.