johnpangalos · johnpangalos · Jul 22, 2026
diff --git a/README.md b/README.md
@@ -89,17 +89,25 @@ reply = client.chat.completions.create(
 
 ## Configuration
 
-| Variable                  | Purpose                                                     |
-| ------------------------- | ----------------------------------------------------------- |
-| `FREELLAMA_HOME`          | Data directory (default `~/.freellama`)                     |
-| `FREELLAMA_CTX`           | Context size passed to llama-server (default `4096`)        |
-| `FREELLAMA_LLAMA_VERSION` | Pin a llama.cpp release tag, e.g. `b5900` (default: latest) |
-| `FREELLAMA_LLAMA_SERVER`  | Path to an existing `llama-server` binary (skips downloads) |
-| `FREELLAMA_SERVER_ARGS`   | Extra flags passed through to `llama-server`                |
-| `FREELLAMA_DEBUG=1`       | Show llama-server output for troubleshooting                |
-| `HF_TOKEN`                | Hugging Face token for gated model repos                    |
-
-Models are stored in `~/.freellama/models`, llama.cpp binaries in `~/.freellama/bin/<tag>`.
+| Variable                  | Purpose                                                              |
+| ------------------------- | -------------------------------------------------------------------- |
+| `FREELLAMA_HOME`          | Data directory (default `~/.freellama`)                              |
+| `FREELLAMA_CTX`           | Context size passed to llama-server (default `4096`)                 |
+| `FREELLAMA_BACKEND`       | llama.cpp build variant: `vulkan`, `cuda`, `rocm`, ... (default CPU) |
+| `FREELLAMA_LLAMA_VERSION` | Pin a llama.cpp release tag, e.g. `b5900` (default: latest)          |
+| `FREELLAMA_LLAMA_SERVER`  | Path to an existing `llama-server` binary (skips downloads)          |
+| `FREELLAMA_SERVER_ARGS`   | Extra flags passed through to `llama-server`                         |
+| `FREELLAMA_READY_TIMEOUT` | Seconds to wait for llama-server to load a model (default `180`)     |
+| `FREELLAMA_DEBUG=1`       | Show llama-server output for troubleshooting                         |
+| `HF_TOKEN`                | Hugging Face token for gated model repos                             |
+
+`FREELLAMA_BACKEND` picks the matching GPU build from llama.cpp's prebuilt releases (e.g.
+`FREELLAMA_BACKEND=vulkan` for AMD/Intel GPUs, `cuda` for NVIDIA on Windows). Combine it with
+`FREELLAMA_SERVER_ARGS` to offload work onto the GPU, e.g. `FREELLAMA_SERVER_ARGS="-ngl 99 -fa on"`.
+Interrupted model downloads resume where they left off when you re-run `pull`.
+
+Models are stored in `~/.freellama/models`, llama.cpp binaries in `~/.freellama/bin/<tag>` (GPU
+variants in `~/.freellama/bin/<tag>-<backend>`).
 
 ## Development
 

diff --git a/src/lib/backend.ts b/src/lib/backend.ts
@@ -35,28 +35,40 @@ const GPU_TOKENS = ["cuda", "vulkan", "hip", "rocm", "sycl", "kompute", "opencl"
 // llama.cpp ships Windows builds as .zip and macOS/Linux builds as .tar.gz.
 const ARCHIVE_RE = /\.(zip|tar\.gz|tgz)$/i;
 
+/** Requested backend variant ("cpu" and unset mean the plain CPU/Metal build). */
+function requestedBackend(): string | undefined {
+  const backend = Deno.env.get("FREELLAMA_BACKEND")?.trim().toLowerCase();
+  return backend && backend !== "cpu" ? backend : undefined;
+}
+
 /**
- * Pick the best CPU/Metal release asset for an OS/arch. Asset names look like
+ * Pick the best release asset for an OS/arch. Asset names look like
  * "llama-b5900-bin-ubuntu-x64.tar.gz", "llama-b5900-bin-macos-arm64.tar.gz",
- * "llama-b5900-bin-win-cpu-x64.zip". Exported for tests.
+ * "llama-b5900-bin-win-cpu-x64.zip", "llama-b5900-bin-ubuntu-vulkan-x64.tar.gz".
+ * Without `backend`, prefers the plain CPU/Metal build; with a backend token
+ * (e.g. "vulkan", "cuda", "rocm"), only assets carrying that token qualify.
+ * Exported for tests.
  */
 export function pickAsset(
   assets: ReleaseAsset[],
   os: typeof Deno.build.os,
   arch: typeof Deno.build.arch,
+  backend?: string,
 ): ReleaseAsset | undefined {
   const osToken = os === "darwin" ? "macos" : os === "windows" ? "win" : "ubuntu";
   const archToken = arch === "aarch64" ? "arm64" : "x64";
   const candidates = assets.filter((a) => {
     const n = a.name.toLowerCase();
     return ARCHIVE_RE.test(n) && n.includes("-bin-") && n.includes(osToken) &&
-      n.includes(archToken);
+      n.includes(archToken) && (!backend || n.includes(backend));
   });
   const score = (a: ReleaseAsset): number => {
     const n = a.name.toLowerCase();
     let s = 0;
-    if (n.includes("cpu")) s += 2;
-    for (const gpu of GPU_TOKENS) if (n.includes(gpu)) s -= 5;
+    if (!backend) {
+      if (n.includes("cpu")) s += 2;
+      for (const gpu of GPU_TOKENS) if (n.includes(gpu)) s -= 5;
+    }
     // Tie-breaker: prefer the plainest build (fewest descriptor segments).
     s -= n.split("-").length * 0.1;
     return s;
@@ -76,12 +88,23 @@ async function fetchRelease(version: string): Promise<Release> {
   return (await resp.json()) as Release;
 }
 
-async function findInstalled(): Promise<string | undefined> {
+// Backend builds install into "<tag>-<backend>" so variants of the same tag
+// don't collide; the plain CPU/Metal build keeps the bare "<tag>" dir.
+function installDirName(tag: string, backend: string | undefined): string {
+  return backend ? `${tag}-${backend}` : tag;
+}
+
+function dirMatchesBackend(dir: string, backend: string | undefined): boolean {
+  if (backend) return dir.endsWith(`-${backend}`);
+  return !GPU_TOKENS.some((gpu) => dir.includes(`-${gpu}`));
+}
+
+async function findInstalled(backend: string | undefined): Promise<string | undefined> {
   const exe = Deno.build.os === "windows" ? "llama-server.exe" : "llama-server";
   try {
     const tags: string[] = [];
     for await (const entry of Deno.readDir(binDir())) {
-      if (entry.isDirectory) tags.push(entry.name);
+      if (entry.isDirectory && dirMatchesBackend(entry.name, backend)) tags.push(entry.name);
     }
     // Rolling llama.cpp tags ("b5900") sort correctly by numeric part.
     tags.sort((a, b) => b.localeCompare(a, undefined, { numeric: true }));
@@ -153,6 +176,7 @@ async function extractTarGz(archive: Uint8Array, installDir: string): Promise<vo
  * (or latest) llama.cpp release if needed. Returns the absolute binary path.
  *
  * Set FREELLAMA_LLAMA_VERSION to pin a release tag (e.g. "b5900");
+ * FREELLAMA_BACKEND to pick a GPU build variant (e.g. "vulkan", "cuda", "rocm");
  * FREELLAMA_LLAMA_SERVER to point at an existing llama-server binary and skip
  * downloads entirely.
  */
@@ -161,27 +185,34 @@ export async function ensureLlamaServer(): Promise<string> {
   if (explicit) return explicit;
 
   const version = Deno.env.get("FREELLAMA_LLAMA_VERSION") ?? "latest";
+  const backend = requestedBackend();
   const exe = Deno.build.os === "windows" ? "llama-server.exe" : "llama-server";
 
   // Without an explicit pin, reuse whatever is already installed before going online.
   if (version === "latest") {
-    const installed = await findInstalled();
+    const installed = await findInstalled(backend);
     if (installed) return installed;
   } else {
-    const existing = await findFile(join(binDir(), version), exe);
+    const existing = await findFile(join(binDir(), installDirName(version, backend)), exe);
     if (existing) return existing;
   }
 
   const release = await fetchRelease(version);
-  const asset = pickAsset(release.assets, Deno.build.os, Deno.build.arch);
+  const asset = pickAsset(release.assets, Deno.build.os, Deno.build.arch, backend);
   if (!asset) {
     throw new Error(
-      `No prebuilt llama.cpp binary for ${Deno.build.os}/${Deno.build.arch} in release ${release.tag_name}. ` +
-        `Build llama.cpp yourself and set FREELLAMA_LLAMA_SERVER to the llama-server path.`,
+      `No prebuilt llama.cpp binary for ${Deno.build.os}/${Deno.build.arch}${
+        backend ? ` with backend "${backend}"` : ""
+      } in release ${release.tag_name}. ` +
+        (backend
+          ? `Available assets:\n${release.assets.map((a) => `  - ${a.name}`).join("\n")}\n` +
+            `Pick a FREELLAMA_BACKEND matching one of these, or build llama.cpp yourself ` +
+            `and set FREELLAMA_LLAMA_SERVER to the llama-server path.`
+          : `Build llama.cpp yourself and set FREELLAMA_LLAMA_SERVER to the llama-server path.`),
     );
   }
 
-  const installDir = join(binDir(), release.tag_name);
+  const installDir = join(binDir(), installDirName(release.tag_name, backend));
   const progress = progressPrinter(`downloading llama.cpp ${release.tag_name} (${asset.name})`);
   const resp = await fetch(asset.browser_download_url, { headers: githubHeaders() });
   if (!resp.ok || !resp.body) {

diff --git a/src/lib/hf.ts b/src/lib/hf.ts
@@ -176,8 +176,10 @@ export interface DownloadProgress {
 }
 
 /**
- * Download a GGUF to the models directory. Idempotent: skips the download when the
- * file already exists with the expected size. Returns the local path.
+ * Download a GGUF to the models directory. Idempotent: skips the download when
+ * the file already exists with the expected size. A leftover ".partial" file
+ * from an interrupted download is resumed with an HTTP Range request rather
+ * than restarted. Returns the local path.
  */
 export async function downloadGguf(
   repo: string,
@@ -195,30 +197,54 @@ export async function downloadGguf(
 
   await Deno.mkdir(modelsDir(), { recursive: true });
   const url = `${HF_BASE}/${repo}/resolve/main/${remotePath}?download=true`;
-  const resp = await fetch(url, { headers: authHeaders() });
+
+  const tmp = dest + ".partial";
+  let offset = 0;
+  try {
+    const stat = await Deno.stat(tmp);
+    if (stat.size > 0 && stat.size < expectedSize) offset = stat.size;
+  } catch (err) {
+    if (!(err instanceof Deno.errors.NotFound)) throw err;
+  }
+
+  const headers = new Headers(authHeaders());
+  if (offset > 0) headers.set("Range", `bytes=${offset}-`);
+  const resp = await fetch(url, { headers });
+  // A server that ignores the Range request replies 200 with the full body.
+  if (offset > 0 && resp.status !== 206) offset = 0;
   if (!resp.ok || !resp.body) {
+    await resp.body?.cancel();
+    // Range no longer satisfiable (e.g. the remote file changed): the stale
+    // partial would fail the same way forever, so drop it before giving up.
+    if (resp.status === 416) await Deno.remove(tmp).catch(() => {});
     throw new Error(`Download failed: HTTP ${resp.status} for ${url}`);
   }
-  const total = Number(resp.headers.get("content-length")) || expectedSize || undefined;
+  const total = expectedSize ||
+    (Number(resp.headers.get("content-length")) + offset) || undefined;
 
-  const tmp = dest + ".partial";
-  let received = 0;
+  let received = offset;
   const counter = new TransformStream<Uint8Array, Uint8Array>({
     transform(chunk, controller) {
       received += chunk.byteLength;
       onProgress?.({ received, total });
       controller.enqueue(chunk);
     },
   });
-  const file = await Deno.open(tmp, { write: true, create: true, truncate: true });
+  const file = await Deno.open(tmp, {
+    write: true,
+    create: true,
+    truncate: offset === 0,
+    append: offset > 0,
+  });
   await resp.body.pipeThrough(counter).pipeTo(file.writable);
-  // A cleanly-closed-but-truncated response must not be recorded as a valid model.
+  // A cleanly-closed-but-truncated response must not be recorded as a valid
+  // model; keep the .partial so the next attempt resumes it.
   if (expectedSize > 0 && received !== expectedSize) {
-    await Deno.remove(tmp).catch(() => {});
+    if (received > expectedSize) await Deno.remove(tmp).catch(() => {});
     throw new Error(
       `Download of ${remotePath} incomplete: got ${formatBytes(received)}, expected ${
         formatBytes(expectedSize)
-      }. Try again.`,
+      }. Try again${received < expectedSize ? " to resume" : ""}.`,
     );
   }
   await Deno.rename(tmp, dest);

diff --git a/src/lib/runner.ts b/src/lib/runner.ts
@@ -24,12 +24,23 @@ function freePort(): number {
   return port;
 }
 
-const READY_TIMEOUT_MS = 180_000;
+const DEFAULT_READY_TIMEOUT_S = 180;
+
+/**
+ * Seconds to wait for llama-server to become ready, from
+ * FREELLAMA_READY_TIMEOUT. Large models (tens of GB) can take longer than the
+ * default 180 s to load from disk. Exported for tests.
+ */
+export function readyTimeoutSeconds(): number {
+  const raw = Number(Deno.env.get("FREELLAMA_READY_TIMEOUT"));
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_READY_TIMEOUT_S;
+}
 
 export async function startLlamaServer(opts: StartOptions): Promise<LlamaServerHandle> {
   const port = freePort();
   const debug = Deno.env.get("FREELLAMA_DEBUG") === "1";
   const contextSize = opts.contextSize ?? Number(Deno.env.get("FREELLAMA_CTX") ?? 4096);
+  const readyTimeoutS = readyTimeoutSeconds();
 
   const args = [
     "-m",
@@ -77,12 +88,15 @@ export async function startLlamaServer(opts: StartOptions): Promise<LlamaServerH
         }
       },
       (healthy) => healthy,
-      { interval: 300, signal: AbortSignal.timeout(READY_TIMEOUT_MS) },
+      { interval: 300, signal: AbortSignal.timeout(readyTimeoutS * 1000) },
     );
   } catch (err) {
     if (err instanceof DOMException && err.name === "TimeoutError") {
       proc.kill("SIGKILL");
-      throw new Error(`llama-server did not become ready within ${READY_TIMEOUT_MS / 1000}s`);
+      throw new Error(
+        `llama-server did not become ready within ${readyTimeoutS}s. ` +
+          `Large models can take longer to load; raise FREELLAMA_READY_TIMEOUT (seconds).`,
+      );
     }
     throw err;
   }

diff --git a/tests/integration_test.ts b/tests/integration_test.ts
@@ -5,9 +5,10 @@
 import { assert, assertEquals } from "@std/assert";
 import { poll } from "@std/async";
 import { fromFileUrl, join } from "@std/path";
-import { startLlamaServer } from "../src/lib/runner.ts";
+import { readyTimeoutSeconds, startLlamaServer } from "../src/lib/runner.ts";
 import { streamChat } from "../src/lib/openai.ts";
 import { pullModel } from "../src/commands/pull.ts";
+import { downloadGguf, localPathFor } from "../src/lib/hf.ts";
 import { getModel, removeModel } from "../src/lib/store.ts";
 
 const projectRoot = fromFileUrl(new URL("..", import.meta.url));
@@ -97,6 +98,75 @@ Deno.test("pull downloads every part of a split gguf and rm removes them all", a
   }
 });
 
+Deno.test("downloadGguf resumes an interrupted download with a Range request", async () => {
+  const home = await Deno.makeTempDir({ prefix: "freellama-resume-" });
+  const prevHome = Deno.env.get("FREELLAMA_HOME");
+  Deno.env.set("FREELLAMA_HOME", home);
+  const realFetch = globalThis.fetch;
+
+  const full = new TextEncoder().encode("0123456789abcdef");
+  let sawRange: string | null = null;
+  globalThis.fetch = ((input: URL | Request | string, init?: RequestInit) => {
+    const url = String(input instanceof Request ? input.url : input);
+    if (!url.includes("resolve/main/")) return realFetch(input, init);
+    sawRange = new Headers(init?.headers).get("Range");
+    const match = sawRange?.match(/^bytes=(\d+)-$/);
+    if (match) {
+      return Promise.resolve(
+        new Response(full.slice(Number(match[1])), {
+          status: 206,
+          headers: { "Content-Range": `bytes ${match[1]}-15/16` },
+        }),
+      );
+    }
+    return Promise.resolve(new Response(full));
+  }) as typeof fetch;
+
+  try {
+    // Leftover partial from an interrupted download: the first 6 bytes.
+    await Deno.mkdir(join(home, "models"), { recursive: true });
+    const dest = localPathFor("user/repo", "m.gguf");
+    await Deno.writeFile(dest + ".partial", full.slice(0, 6));
+
+    const path = await downloadGguf("user/repo", "m.gguf", full.byteLength);
+    assertEquals(sawRange, "bytes=6-");
+    assertEquals(await Deno.readTextFile(path), "0123456789abcdef");
+
+    // A server that ignores Range restarts the download from scratch.
+    await Deno.remove(dest);
+    await Deno.writeFile(dest + ".partial", full.slice(0, 6));
+    globalThis.fetch = ((input: URL | Request | string, init?: RequestInit) => {
+      const url = String(input instanceof Request ? input.url : input);
+      if (!url.includes("resolve/main/")) return realFetch(input, init);
+      return Promise.resolve(new Response(full));
+    }) as typeof fetch;
+    const restarted = await downloadGguf("user/repo", "m.gguf", full.byteLength);
+    assertEquals(await Deno.readTextFile(restarted), "0123456789abcdef");
+  } finally {
+    globalThis.fetch = realFetch;
+    if (prevHome === undefined) Deno.env.delete("FREELLAMA_HOME");
+    else Deno.env.set("FREELLAMA_HOME", prevHome);
+    await Deno.remove(home, { recursive: true });
+  }
+});
+
+Deno.test("readyTimeoutSeconds honors FREELLAMA_READY_TIMEOUT", () => {
+  const prev = Deno.env.get("FREELLAMA_READY_TIMEOUT");
+  try {
+    Deno.env.delete("FREELLAMA_READY_TIMEOUT");
+    assertEquals(readyTimeoutSeconds(), 180);
+    Deno.env.set("FREELLAMA_READY_TIMEOUT", "900");
+    assertEquals(readyTimeoutSeconds(), 900);
+    Deno.env.set("FREELLAMA_READY_TIMEOUT", "not-a-number");
+    assertEquals(readyTimeoutSeconds(), 180);
+    Deno.env.set("FREELLAMA_READY_TIMEOUT", "-5");
+    assertEquals(readyTimeoutSeconds(), 180);
+  } finally {
+    if (prev === undefined) Deno.env.delete("FREELLAMA_READY_TIMEOUT");
+    else Deno.env.set("FREELLAMA_READY_TIMEOUT", prev);
+  }
+});
+
 Deno.test("runner starts the server, streamChat streams, stop terminates", async () => {
   const { home, wrapper } = await makeFixture();
   try {

diff --git a/tests/unit_test.ts b/tests/unit_test.ts
@@ -95,6 +95,26 @@ Deno.test("pickAsset: windows prefers cpu build over cuda", () => {
   assertEquals(pickAsset(assets, "windows", "x86_64")?.name, "llama-b10068-bin-win-cpu-x64.zip");
 });
 
+Deno.test("pickAsset: backend selects the matching GPU variant", () => {
+  assertEquals(
+    pickAsset(assets, "linux", "x86_64", "vulkan")?.name,
+    "llama-b10068-bin-ubuntu-vulkan-x64.tar.gz",
+  );
+  assertEquals(
+    pickAsset(assets, "linux", "x86_64", "rocm")?.name,
+    "llama-b10068-bin-ubuntu-rocm-7.2-x64.tar.gz",
+  );
+  assertEquals(
+    pickAsset(assets, "windows", "x86_64", "cuda")?.name,
+    "llama-b10068-bin-win-cuda-12.4-x64.zip",
+  );
+});
+
+Deno.test("pickAsset: unavailable backend matches nothing", () => {
+  assertEquals(pickAsset(assets, "linux", "x86_64", "cuda"), undefined);
+  assertEquals(pickAsset(assets, "darwin", "aarch64", "vulkan"), undefined);
+});
+
 Deno.test("pickAsset: legacy .zip macOS assets still match", () => {
   const legacy = [{
     name: "llama-b5900-bin-macos-arm64.zip",