Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions src/lib/inference/llama-cpp/managed-installer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1050,6 +1050,77 @@ describe("managed llama.cpp installer", () => {
expect(lifecycle.start).toHaveBeenCalledOnce();
});

it.each([
["authentication", { status: 1, stderr: "unauthorized: authentication required" }],
["storage", { status: 1, stderr: "no space left on device" }],
["runner network", { status: 1, stderr: "dial tcp: temporary failure in name resolution" }],
["invalid dependency", { status: 1, stderr: "manifest unknown: manifest not found" }],
["registry availability", { status: 1, stderr: "registry returned 503 Service Unavailable" }],
["daemon behavior", { status: 1, error: new Error("Cannot connect to the Docker daemon") }],
["unclassified", { status: 7, stderr: "opaque pull failure" }],
])(
"classifies a failed image pull from diagnostic signatures as %s (#10558)",
async (layer, pullResult) => {
const selected = selection();
const homeDir = temporaryHome();
const harness = engineHarness();

const result = await installManagedLlamaCpp(selected, {
sandboxName: "spark-agent",
homeDir,
runtimeProvider: managedRuntimeProvider(harness.engine),
pullImage: vi.fn(async () => pullResult),
verifyGguf: vi.fn(async () => {
throw new Error("not cached");
}),
checkPort: vi.fn(async () => ({ ok: true })),
log: vi.fn(),
});

const evidence =
layer === "unclassified"
? "No recognized failure-layer signature matched."
: "A pull-output signature matched this layer.";
expect(result).toMatchObject({
ok: false,
reason: expect.stringContaining(`Failure classification: ${layer}. ${evidence}`),
});
},
);

it("reports a bounded image-pull classification without streaming sensitive output (#10558)", async () => {
const selected = selection();
const homeDir = temporaryHome();
const harness = engineHarness();
const secret = "opaque-pull-token";
const pullOutput =
`unauthorized: https://pull-user:${secret}@registry.example/v2/image?token=${secret} ` +
"x".repeat(1_000);
const log = vi.fn();
const result = await installManagedLlamaCpp(selected, {
sandboxName: "spark-agent",
homeDir,
runtimeProvider: managedRuntimeProvider(harness.engine),
pullImage: vi.fn(async (_image, options) => {
options.logLine(pullOutput);
return { status: 1, stderr: pullOutput };
}),
verifyGguf: vi.fn(async () => {
throw new Error("not cached");
}),
checkPort: vi.fn(async () => ({ ok: true })),
log,
});

const failure = result as Extract<typeof result, { readonly ok: false }>;
expect(failure.reason).toContain("Failure classification: authentication.");
expect(failure.reason).toContain("Raw pull output is not included.");
expect(failure.reason).not.toContain(secret);
expect(Buffer.byteLength(failure.reason, "utf8")).toBeLessThan(400);
expect(log.mock.calls.flat().join("\n")).not.toContain(secret);
expect(log.mock.calls.flat().join("\n")).not.toContain("unauthorized");
});

it("resumes an exact cached runtime without image pulls or Hugging Face acquisition", async () => {
const selected = selection();
const homeDir = temporaryHome();
Expand Down
73 changes: 71 additions & 2 deletions src/lib/inference/llama-cpp/managed-installer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ interface DockerNetworkInspection {
interface ManagedLlamaCppImagePullResult {
readonly status: number | null;
readonly error?: Error;
readonly stderr?: string;
readonly stdout?: string;
}

type ManagedLlamaCppImagePull = (
Expand All @@ -132,6 +134,71 @@ type ManagedLlamaCppImagePull = (
},
) => Promise<ManagedLlamaCppImagePullResult>;

type ManagedLlamaCppImagePullFailureLayer =
| "authentication"
| "daemon behavior"
| "invalid dependency"
| "registry availability"
| "runner network"
| "storage"
| "unclassified";

const IMAGE_PULL_LAYER_PATTERNS: ReadonlyArray<{
readonly layer: ManagedLlamaCppImagePullFailureLayer;
readonly pattern: RegExp;
}> = [
{
layer: "authentication",
pattern:
/(?:authentication required|unauthorized|pull access denied|requested access .* denied|credential helper|credentials? store)/iu,
},
{
layer: "storage",
pattern: /(?:no space left on device|disk quota exceeded|insufficient disk space)/iu,
},
{
layer: "runner network",
pattern:
/(?:dial tcp|no such host|temporary failure in name resolution|network is unreachable|connection (?:refused|reset)|tls handshake timeout|i\/o timeout|context deadline exceeded|client\.timeout|certificate signed by unknown authority)/iu,
},
{
layer: "invalid dependency",
pattern:
/(?:manifest unknown|manifest .* not found|no matching manifest|no match for platform)/iu,
},
{
layer: "registry availability",
pattern:
/(?:too many requests|rate limit|service unavailable|bad gateway|gateway timeout|status(?: code)?:? 5\d\d|unexpected status.*5\d\d)/iu,
},
{
layer: "daemon behavior",
pattern:
/(?:cannot connect to the (?:docker|container) daemon|daemon is not running|error during connect|docker pull failed to start)/iu,
},
];

function classifyImagePullFailureLayer(diagnostic: string): ManagedLlamaCppImagePullFailureLayer {
return (
IMAGE_PULL_LAYER_PATTERNS.find(({ pattern }) => pattern.test(diagnostic))?.layer ??
"unclassified"
);
}

function imagePullFailureDiagnostic(result: ManagedLlamaCppImagePullResult): string {
const sources = [result.stdout, result.stderr, result.error?.message].filter(
(value): value is string => typeof value === "string" && value.trim().length > 0,
);
const layer = classifyImagePullFailureLayer(sources.join("\n"));
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Preserve daemon-error precedence. The classifier returns the first matching layer, but runner network is checked before daemon behavior. Real Docker errors commonly combine error during connect with dial tcp or connection refused, so those daemon failures are mislabeled as runner network. Move the daemon-specific rule ahead of the broad network rule, or encode explicit precedence, and add a mixed-signal regression that expects daemon behavior.

const evidence =
layer === "unclassified"
? "No recognized failure-layer signature matched."
: "A pull-output signature matched this layer.";
const status =
result.status === null ? "Exit status unavailable." : `Exit status: ${String(result.status)}.`;
return `Failure classification: ${layer}. ${evidence} ${status} Raw pull output is not included.`;
}

function requireSuccess(label: string, result: ReturnType<ContainerEngine["capture"]>): string {
if (result.error || result.status !== 0) {
throw new Error(`Managed llama.cpp ${label} failed (exit ${String(result.status)}).`);
Expand Down Expand Up @@ -358,11 +425,13 @@ async function pullExactImages(
? await pull(image, {
env: dockerEnv,
maxTimeoutMs: IMAGE_PULL_TIMEOUT_MS,
logLine: (line) => log(` ${line}`),
logLine: () => undefined,
})
: engine.capture(["pull", image], IMAGE_PULL_TIMEOUT_MS);
if (result.status !== 0 || ("error" in result && result.error !== undefined)) {
throw new Error(`Pinned managed-inference image pull failed for ${image}.`);
throw new Error(
`Pinned managed-inference image pull failed for ${image}. ${imagePullFailureDiagnostic(result)}`,
);
}
const after = engine.capture(["image", "inspect", image], DOCKER_INSPECT_TIMEOUT_MS);
if (after.error || after.status !== 0) {
Expand Down
Loading
Loading