feat(llmkube): tune vllm staging config and document VRAM ceiling

docs/llm-hosting/bench/concsweep.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Measure the real decode ceiling by concurrency sweep.
+
+Short prompts on purpose: isolates DECODE from prefill. Unique salt per stream so
+prefix caching cannot fake the numbers. Aggregate tok/s is the number that matters -
+the owner cares about concurrency, not batch-1 latency.
+"""
+import json, sys, time, threading, urllib.request
+
+def _port():
+    if len(sys.argv) < 2:
+        return 8000
+    p = int(sys.argv[1])
+    if not 1 <= p <= 65535:
+        raise SystemExit(f"port out of range: {p}")
+    return p
+
+
+URL = f"http://127.0.0.1:{_port()}/v1/completions"
+MODEL = "qwen-3.6"
+GEN = 96
+
+
+def stream(salt, out, lk):
+    body = json.dumps({
+        "model": MODEL,
+        "prompt": f"Run {salt}. Write a long detailed essay about distributed storage systems.",
+        "max_tokens": GEN, "temperature": 0.7, "ignore_eos": True,
+    }).encode()
+    t0 = time.perf_counter()
+    try:
+        rq = urllib.request.Request(URL, data=body, headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(rq, timeout=1200) as r:
+            d = json.load(r)
+        el = time.perf_counter() - t0
+        n = d["usage"]["completion_tokens"]
+        with lk:
+            out.append((n, el))
+    except Exception as e:
+        with lk:
+            out.append((0, time.perf_counter() - t0, str(e)[:80]))
+
+
+print(f"{'conc':>5} {'agg tok/s':>10} {'per-stream':>11} {'wall s':>8} {'ok':>4}")
+for c in [1, 8, 16]:
+    out, lk, th = [], threading.Lock(), []
+    t0 = time.perf_counter()
+    for i in range(c):
+        t = threading.Thread(target=stream, args=(f"c{c}s{i}", out, lk))
+        t.start()
+        th.append(t)
+    for t in th:
+        t.join()
+    wall = time.perf_counter() - t0
+    ok = [o for o in out if o[0] > 0]
+    tot = sum(o[0] for o in ok)
+    agg = tot / wall if wall else 0
+    per = agg / len(ok) if ok else 0
+    print(f"{c:>5} {agg:>10.2f} {per:>11.2f} {wall:>8.1f} {len(ok):>4}", flush=True)
+    time.sleep(4)

docs/llm-hosting/bench/spectest.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""Realistic spec-decode test: long prompt the output must quote back verbatim.
+This is the coding-agent shape (resend file, ask for an edit) where prompt-lookup
+n-gram should hit. A short-prompt/novel-output test would show ~zero acceptance.
+"""
+import json, time, urllib.request, sys
+def _port():
+    if len(sys.argv) < 2:
+        return 8000
+    p = int(sys.argv[1])
+    if not 1 <= p <= 65535:
+        raise SystemExit(f"port out of range: {p}")
+    return p
+
+
+URL = f"http://127.0.0.1:{_port()}/v1/completions"
+CODE = "\n".join(f"def handler_{i}(request, context):\n"
+                 f"    payload = request.get('payload_{i}')\n"
+                 f"    if payload is None:\n"
+                 f"        raise ValueError('missing payload_{i}')\n"
+                 f"    return {{'status': 200, 'body': payload}}\n" for i in range(240))
+prompt = (f"Here is a Python module:\n\n{CODE}\n\n"
+          "Reproduce handler_0 through handler_12 EXACTLY as written above, verbatim:\n\n")
+b = json.dumps({"model": "qwen-3.6", "prompt": prompt, "max_tokens": 400,
+                "temperature": 0, "ignore_eos": True}).encode()
+t0 = time.perf_counter()
+rq = urllib.request.Request(URL, data=b, headers={"Content-Type": "application/json"})
+with urllib.request.urlopen(rq, timeout=900) as r:
+    d = json.load(r)
+el = time.perf_counter() - t0
+u = d["usage"]
+print(f"prompt={u['prompt_tokens']} gen={u['completion_tokens']} wall={el:.1f}s "
+      f"TG={u['completion_tokens']/el:.2f} tok/s")
+
+# The throughput number is only meaningful if the model actually quoted back the
+# source. GEN=400 cannot cover all 13 handlers, so a correct run is normally a
+# clean prefix; only an actual mismatch invalidates the measurement.
+expect = "\n".join(CODE.split("\n\n")[:13]).strip()
+got = d["choices"][0]["text"].strip()
+if got and expect.startswith(got):
+    state = "OK" if got == expect else "TRUNCATED"
+else:
+    state = "DIVERGED"
+n = next((i for i, (a, b) in enumerate(zip(expect, got)) if a != b),
+         min(len(expect), len(got)))
+print(f"verbatim: {state} at char {n}/{len(expect)}")
+print("---- first 200 chars of output ----")
+print(got[:200])

docs/llm-hosting/vllm-vs-sglang-2026-07.md

@@ -0,0 +1,197 @@
+# vLLM vs SGLang on the R9700 — 2026-07 measurement round
+
+Qwen3.6-27B AWQ on a single Radeon AI PRO R9700 (gfx1201, 32 GB). Records what was
+measured, what it overturned, and why the current configs look the way they do.
+
+## The binding constraint: 0.875 VRAM
+
+**The R9700 is shared with Jellyfin media transcoding.** LLM serving must never exceed
+`0.875` (`memFractionStatic` for SGLang, `gpuMemoryUtilization` for vLLM). The ~4.6 GB
+`rocm-smi` reports free at 0.875 is the transcoding reservation, not slack.
+
+Any benchmark taken above 0.875 is invalid as a production candidate no matter how well
+it scores. Two were run during this round before the constraint was known; both are
+recorded below and both are rejected.
+
+## What was overturned
+
+### The "decode ceiling" was KV-pool starvation
+
+Earlier rounds concluded aggregate throughput plateaued near 40 tok/s and that this was a
+fixed per-step overhead, not bandwidth. Half right: the plateau was real, the cause was
+not the GPU.
+
+At concurrency 16 under a small pool, per-stream throughput collapsed to 2.45 tok/s (from
+4.85 at concurrency 8) because requests were queuing and preempting. Doubling the pool
+held per-stream at ~4.6 and aggregate nearly doubled:
+
+| pool | conc 8 | conc 16 | per-stream @16 |
+|---|---|---|---|
+| 109,217 tok | 39.04 | 39.32 | 2.45 |
+| 213,362 tok | 39.34 | **74.50 / 73.84** | 4.62 |
+
+The 213K row was taken at 0.95 and is **not deployable**. The finding it produced is
+still valid and redirects tuning effort: grow the KV pool within 0.875, do not chase
+kernel-level decode optimisations.
+
+### Benchmark runs are unreliable immediately after boot
+
+The first sweep after a cold start reads low — one config measured 32.36 tok/s at
+concurrency 16 on its first run and 39.04 / 39.09 on two immediate repeats. Historic
+"noise" of 32.8–38.5 at concurrency 8 was largely this artifact, and some earlier
+single-run comparisons were reading warmup as a config difference.
+
+**Always discard the first run after a pod restart.**
+
+### Image tags are not ordered by version number
+
+The `vllm-openai-rocm` nightly self-reports `0.23.1rc1.dev1474+g0ba2aa35a` — a stale base
+tag 1474 commits behind its actual content. The `v0.26.0` release was *built earlier*
+(02:47Z) than the nightly (05:17Z) despite the higher version.
+
+Neither publish timestamps nor tag names establish lineage. Only the self-reported
+version string plus build metadata do.
+
+## Engine comparison
+
+Same sweep script, short prompts to isolate decode, unique salt per stream so prefix
+caching cannot inflate results.
+
+At the production 0.875 budget, SGLang serves a **183,240**-token GPU pool plus 9.01 GB
+host-RAM hierarchical KV and 7.14 GB host mamba cache, with 3.71 GB GPU headroom left for
+transcoding. It clamps `max_running_requests` 32 → 20 at this size.
+
+The table below is **0.95-only**: neither column is deployable, and nothing in it has
+been re-measured at 0.875, so no row should be quoted as a property of the production
+configs.
+
+| | vLLM v0.26.0 | SGLang v0.5.15 |
+|---|---|---|
+| GPU KV pool | 213,362 tok | 261,019 tok |
+| host-RAM L2 | none | 12.8 GB KV + 7.1 GB mamba |
+| conc 1 | 4.28 / 5.17 | **14.18 / 14.44** |
+| conc 8 | **39.34** | 36.28 |
+| conc 16 | **74.50** | 69.46 / 70.60 |
+| 13.8K prefill | served | **OOM, scheduler died** |
+
+Reading the table: SGLang leads single-stream, vLLM leads at concurrency 16, and the
+`13.8K prefill` row is the one that disqualifies the config — SGLang survived short-prompt
+decode and died on the prompt shape we actually serve, while vLLM served it on the same
+budget. That is evidence the 261K pool is unreachable in practice, not evidence about
+0.875 stability; SGLang has run at 0.875 in production for weeks.
+
+Pool size is the only figure comparable across budgets, since it is reported at startup
+rather than benchmarked. **A like-for-like engine comparison at 0.875 has not been run**
+and is the main gap in this round.
+
+## Speculative decoding (vLLM, n-gram k=3)
+
+Accepted 286 of 303 draft tokens (94.4%). Helps repetition-heavy work and hurts
+concurrency:
+
+| workload | baseline | ngram |
+|---|---|---|
+| quote-back (13.8K prompt) | 4.06 | **7.52** (+85%) |
+| novel generation, conc 8 | 32.82 | 37.84 (+15%) |
+| novel generation, conc 16 | **64.96** | 39.57 (−39%) |
+
+It also force-disables async scheduling and costs ~12K tokens of KV pool. **Left out of
+the committed config** because concurrency is the priority for this workload. Worth
+revisiting if single-stream latency becomes the dominant complaint.
+
+## SGLang v0.5.16
+
+Skip the release, cherry-pick one commit. The release relocated the kernel tree
+(`srt/layers/attention/**` → `kernels/ops/**`), which breaks the fork's ~90 patches.
+
+`#31648` (mamba LRU) is the one worth taking: prefix hit rate 0.61 → 0.83. Not yet
+applied — it needs a fork rebuild.
+
+## Operational hazard: patching during a model download corrupts the cache
+
+Hit during this round and worth knowing before it costs someone else 40 minutes.
+
+Patching an `InferenceService` while its `model-downloader` init container is mid-transfer
+replaces the pod and truncates the file being written. The corruption is then **invisible
+to every subsequent boot** until the engine tries to load the weights and fails with
+`SafetensorError: incomplete metadata, file not fully covered`.
+
+The downloader runs:
+
+```sh
+curl -fsSL --etag-compare "$etag" --etag-save "$etag" -o "$dest" "$url"
+```
+
+Three things combine:
+
+1. **No atomic write.** curl streams to the final path, so an interrupted transfer leaves
+   a truncated file where a valid one belongs. A `$dest.tmp` + `mv` would make this
+   impossible.
+2. **ETag is not integrity.** The etag file is written from response headers, which
+   arrive before the body, so it can persist for a transfer that later dies. The next
+   boot sends `If-None-Match`, gets `304`, and prints `Model artifact ... revalidated`
+   over a corrupt file. Nothing checks size against `Content-Length`.
+3. **The fallback keeps bad data.** `elif [ -f "$dest" ]` treats presence as validity.
+
+There is currently no content-integrity mechanism at all: sha256 pinning was rejected on
+the `Model` CRD, leaving only revision pinning.
+
+A `Content-Length` comparison on the finished file is the cheapest mitigation and needs
+no schema change, but it is only a **truncation guard**, not integrity. It would have
+caught this incident and would not catch a corrupted-but-complete transfer. It also does
+nothing about atomicity: the artifact and its `.etag` are still published as two separate
+non-atomic steps, so a crash between them leaves the same inconsistent state. A real fix
+is download-to-temp plus a single rename, with digest or format-level validation before
+publishing.
+
+**Recovery:** scale the InferenceService to 0, mount the model-cache PVC in a throwaway
+root pod, delete the bad artifact *and* its `.<name>.etag`, scale back up. Deleting the
+weight file alone is not enough; the stale etag will short-circuit the re-download.
+
+## Open items
+
+- **Prefix cache reports 0% on vLLM** across 889,464 queries with 0 hits and 0
+  preemptions. Confirmed upstream code bug (`vllm#45238`), not a capacity problem. On a
+  workload of agents resending near-identical 35K contexts this is worth more than any
+  decode tuning.
+- **Quant comparison is unfinished.** vLLM runs QuantTrio AWQ, SGLang runs mattbucci AWQ,
+  so the engine comparison above is engine+quant. `mattbucci/Qwen3.6-27B-AWQ-CT`
+  (compressed-tensors W4A16) is the untested candidate on **vLLM only**: it should bind
+  `RDNAHybridW4A16LinearKernel`, a gfx1201-tuned int4 GEMM for M≤5, which the `auto_awq`
+  path does not. It is not an SGLang candidate — its card describes it as the raw GPTQ
+  output *before* AWQ repack, so it is upstream of the model SGLang already serves, and
+  SGLang's compressed-tensors support is applied mainly to `w8a8_fp8`.
+- **How much VRAM does transcoding actually need?** 0.875 is treated as fixed. Whether it
+  has margin, or is itself too high under concurrent transcode load, has not been
+  measured.
+- **Host-RAM L2 for vLLM** (`kv_offloading_size`, `kv_offloading_backend`, present in
+  v0.26.0) costs no VRAM, which makes it attractive under this constraint. Blocked on
+  node memory: control-1 sits at 97% memory requests and SGLang's equivalent wanted
+  ~20 GB.
+
+## Method notes
+
+Scripts live in `bench/` next to this doc, so the numbers above stay auditable:
+
+- `concsweep.py` — aggregate decode at concurrency 1/8/16. Short prompts to isolate
+  decode from prefill, unique salt per stream so prefix caching cannot inflate results.
+- `spectest.py` — verbatim-reproduction throughput on a 13.8K-token prompt. This is the
+  coding-agent shape where prompt-lookup n-gram actually hits; a short-prompt test shows
+  near-zero acceptance.
+
+Both drive the OpenAI `/v1/completions` endpoint, so they run unmodified against either
+engine. Port-forward the service and pass the port:
+
+```sh
+kubectl -n ai port-forward svc/qwen36-27b 30000:30000 &
+python3 concsweep.py 30000
+python3 spectest.py 30000
+```
+
+**Discard the first run after a pod restart** (see above), and note `concsweep.py` reports
+aggregate tok/s, not per-stream. Every figure in this doc comes from these two scripts
+against a single-replica service with nothing else on the GPU.
+
+Workload profile driving these choices, from litellm `LiteLLM_SpendLogs` over 21 days
+(14,678 requests): prompt p50 35,346 / p90 88,583 / p99 112,050; generation p50 196 /
+p90 2,324; concurrency 4–6. **54.4% of real requests exceed a 32K context.**

kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml

@@ -81,6 +81,9 @@ spec:
     claimName: qwen36-27b-model-cache
   sglangConfig:
     contextLength: 180000
+    # HARD CEILING: the R9700 is shared with Jellyfin transcoding, raising this fails
+    # media workloads. 0.95 grows the pool 183,240 -> 261,019 tokens but OOMs on a
+    # 13.8K-token prefill (2026-07-25).
     memFractionStatic: 0.875
     maxRunningRequests: 32
     chunkedPrefillSize: 8192

kubernetes/apps/ai/llmkube/models/qwen36-27b-vllm.yaml

@@ -31,15 +31,20 @@ kind: Model
 metadata:
   name: qwen36-27b-vllm
 spec:
-  # Pin the model revision for reproducible staging.
-  source: hf://cyankiwi/Qwen3.6-27B-AWQ-INT4@e5cc0400fb2403c437c2c40a7c52fb5ae93fda18
+  # Pin the model revision for reproducible staging. QuantTrio is the boot-validated
+  # variant; the prior cyankiwi INT4 pin was never booted.
+  source: hf://QuantTrio/Qwen3.6-27B-AWQ@9b507bdc9afafb87b7898700cc2a591aa6639461
   format: safetensors
-  quantization: compressed-tensors
+  quantization: AWQ
   files:
-    - model-00001-of-00004.safetensors
-    - model-00002-of-00004.safetensors
-    - model-00003-of-00004.safetensors
-    - model-00004-of-00004.safetensors
+    - model-00001-of-00008.safetensors
+    - model-00002-of-00008.safetensors
+    - model-00003-of-00008.safetensors
+    - model-00004-of-00008.safetensors
+    - model-00005-of-00008.safetensors
+    - model-00006-of-00008.safetensors
+    - model-00007-of-00008.safetensors
+    - model-00008-of-00008.safetensors
     - model.safetensors.index.json
     - config.json
     - configuration.json
@@ -74,16 +79,20 @@ spec:
   runtime: vllm
   modelCache:
     claimName: qwen36-27b-vllm-cache
-  # Use the ROCm nightly with the required AMD kernel support.
-  image: vllm/vllm-openai-rocm:nightly@sha256:bd34cea1fe22f0c9f8fef0e362eac2d8b9292971a114eb95d2cfa1026862f5cd
+  # v0.26.0 measured identical to the nightly on KV pool and throughput (2026-07-25).
+  # The nightly self-reports a stale 0.23.1rc1 base tag; tags are not version-ordered.
+  image: vllm/vllm-openai-rocm:v0.26.0@sha256:5709fafe47123becb2f5e61c32d0b97beff1a629bb40bb753c15464f69a97a18
   replicas: 0
-  # Limit concurrent sequence slots to fit the model's cache budget.
-  parallelSlots: 8
+  # 32 slots doubled aggregate throughput vs 8: 40 -> 79 tok/s at concurrency 16.
+  parallelSlots: 32
   vllmConfig:
-    maxModelLen: 180000
+    # 180000 needs ~6.55 GiB KV against a ~4.6 GiB ceiling at 0.875. 64000 covers 74.1%
+    # of measured production prompts.
+    maxModelLen: 64000
     kvCacheDtype: fp8_e4m3
     enablePrefixCaching: true
     maxNumBatchedTokens: 8192
+    # HARD CEILING: shared with Jellyfin transcoding, never raise.
     gpuMemoryUtilization: 0.875
   env:
     - name: VLLM_ROCM_USE_AITER
@@ -119,6 +128,10 @@ spec:
     - --mamba-ssm-cache-dtype
     - bfloat16
     - --trust-remote-code
+    # vLLM's profiler leaves the pool at 48,000 tokens; explicit sizing gives 121,302
+    # within the same 0.875 budget.
+    - --kv-cache-memory-bytes
+    - "4294967296"
   probeOverrides:
     # Allow ample time for model startup and compilation.
     startup: