From 6a321d082dff20b51201e42200d4254581bc4a2a Mon Sep 17 00:00:00 2001 From: Tanguille <91473554+Tanguille@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:37:07 +0200 Subject: [PATCH 1/5] feat(vllm): philbert g128 quant, 9 GiB KV pool, 22 GiB CPU offload tier Brings git in line with the measured-optimal live config and records the measurements that constrain each value. - philbert g128 replaces cyankiwi g32 (ignore lists match on 311/313 entries; group_size is the only material difference, 1.5 GB smaller, thinking-mode calibrated), pinned to the tokenizer-fix revision - nightly-5a4c8d99 - --kv-cache-memory 7 -> 9 GiB: pool 223,172 -> 287,159 tok, 1.01x -> 1.30x - cpu_bytes_to_use 16 -> 22 GiB, dshm 18 -> 24Gi, memory 32 -> 36Gi - correct the VLLM_ROCM_USE_AITER comment: measured inert for attention Warnings added for two changes that measured as regressions and must not be retried: raising maxModelLen to 262,144 (halves decode) and removing the fs secondary offload tier (halves decode). --- .../ai/llmkube/models/qwen38-27b-vllm.yaml | 51 ++++++++++++------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml index 0890e978df..3ad3bf0a7e 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml @@ -122,7 +122,12 @@ kind: Model metadata: name: qwen38-27b-vllm spec: - source: hf://cyankiwi/Qwen3.8-27B-AWQ-INT4@63768c10df38c0395e12ef49edac1bd539eaeeea + # philbert g128 over cyankiwi g32: ignore lists match on 311/313 entries, so + # group_size is the only material difference. 1.5 GB smaller, and the only + # build documenting thinking-mode calibration (llm-compressor #2680 corrupts + # without it; thinking is on by default here). + # Pinned to the 2026-08-15 tokenizer fix; earlier revisions 400 on image input. + source: hf://philbert440/Qwen3.8-27B-W4A16-AWQ@7908d42a71077a5e4dc458f273682b12dfe384a0 format: safetensors quantization: compressed-tensors refreshPolicy: OnChange @@ -169,8 +174,8 @@ spec: parallelSlots: 16 bindAddress: 0.0.0.0 # Commit-tagged nightly, not the rolling `nightly` tag. - # v0.27.1 is the newest release and predates gfx1201 AITER support. - image: vllm/vllm-openai-rocm:nightly-aa9903490c616dc6871e5acc62cec7bb1e5e9434@sha256:d16558a215144a1a90f3ecfa377559fac4143e74424fa7c4fab2020dd947eb9e + # v0.27.1 is the newest release and predates gfx1201 support. + image: vllm/vllm-openai-rocm:nightly-5a4c8d99242e9e069b604d0e9b969e77f7dd501d@sha256:5ea7099b53cdf9049ddcd446f15d831f32bce06c2822d6f8f228b819f189fc48 modelCache: claimName: qwen38-27b-vllm-model-cache vllmConfig: @@ -188,7 +193,9 @@ spec: # match it after ANY of: --kv-cache-memory, the mamba cache dtypes, the # attention block size, or a vLLM image bump. That last one is the easy # miss -- Renovate opens it as an ordinary reviewed PR. - # The model itself allows 262,144; KV memory is the binding limit. + # The pool now fits 262,144 but DO NOT raise it: 262,144 measured ~half the + # single-stream decode (15.8-17.1 vs 29.7-32.0 tok/s), confirmed by exact + # revert; mechanism unexplained. Production prompts are 47-56K anyway. maxModelLen: 221612 kvCacheDtype: fp8_e4m3 # Confirmed working on this hybrid Mamba/GDN model (~27% hit rate under @@ -204,9 +211,10 @@ spec: # sane value. The media-reserve math lives with that flag, not here. gpuMemoryUtilization: 0.875 env: - # gfx1201 AITER Triton paths (vLLM #43615). Its FP8 linear kernels do - # not apply to this W4A16 checkpoint; the attention-backend reorder - # (ROCM_AITER_UNIFIED_ATTN first) and the GDN linear-attn path do. + # Inert for attention: with a KV connector set, rocm.py:703 rejects + # ROCM_AITER_UNIFIED_ATTN and selects TRITON_ATTN. FP8 linear kernels do + # not apply to W4A16 either. Kept for the GDN linear-attn path. + # Forcing AITER attention measured +84% decode but ~10x worse prefill. - name: VLLM_ROCM_USE_AITER value: "1" - name: HIP_VISIBLE_DEVICES @@ -229,12 +237,12 @@ spec: - name: kv-offload persistentVolumeClaim: claimName: qwen38-27b-vllm-kv-offload - # The CPU offload tier mmaps cpu_bytes_to_use (16Gi) here; the 64M - # default EFAULTs on pre-fault. 18Gi = 16Gi + 2Gi margin. + # The CPU offload tier mmaps cpu_bytes_to_use (22Gi) here; the 64M + # default EFAULTs on pre-fault. 24Gi = 22Gi + 2Gi margin. - name: dshm emptyDir: medium: Memory - sizeLimit: 18Gi + sizeLimit: 24Gi extraVolumeMounts: - name: compile-cache mountPath: /cache @@ -289,18 +297,25 @@ spec: # together, which was not observed but is the number the reserve must cover. # Jellyfin has no transcode concurrency cap, so a third stream would exceed # 2 GiB and evict KV; see the DgpuVramLow alert. Measured free at this - # config: 2.81 GB. + # config: 2.57 GB at peak load (was 2.81 GB at 7 GiB) -- this raise spends + # 0.24 GB of that reserve. Pool 223,172 -> 287,159 tokens, concurrency + # 1.01x -> 1.30x. Retention benefit NOT yet measured. - --kv-cache-memory - - "7516192768" + - "9663676416" # Hierarchical KV cache: CPU tier (matches SGLang's --hicache-ratio sizing # philosophy) + fs tier (SGLang's file backend analogue). Ported from # qwen36-27b-vllm.yaml, WITHOUT that config's --language-model-only — # Hermes' auxiliary.vision block actively uses this model's vision tower, # so it must stay loaded even though that costs some context headroom. - # CPU tier 16Gi: 8Gi measured 80% full in production, i.e. saturated. + # CPU tier 22Gi: the 16Gi tier cascaded 3.5 GB/hour to the fs tier, i.e. + # eviction pressure. Benefit NOT yet validated -- needs a restart-free + # window, since this tier is an emptyDir tmpfs that every restart wipes. + # DO NOT remove the fs secondary tier to save Ceph writes: its 4.5% hit + # rate is not its job. Removing it halved single-stream decode (15.5 vs + # 31 tok/s, exact-revert confirmed) -- it keeps eviction asynchronous. - --kv-transfer-config - >- - {"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":17179869184,"secondary_tiers":[{"type":"fs","root_dir":"/kvoffload","locality":"LOCAL"}]}} + {"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":23622320128,"secondary_tiers":[{"type":"fs","root_dir":"/kvoffload","locality":"LOCAL"}]}} nodeSelector: amd.com/gpu: "true" podSecurityContext: @@ -312,10 +327,10 @@ spec: resources: cpu: "2" # Steady state was 13.3Gi with an 8Gi shm offload tier (tmpfs, so charged - # here); the 16Gi tier projects ~21Gi. Kept at 32Gi for the load spike rather - # than trimmed to the steady state -- an OOMKill mid-load throws away the - # compile cache this node takes ~40min to rebuild. - memory: 32Gi + # here); the 22Gi tier measured 25.1Gi total. Headroom kept for the load + # spike rather than trimmed to steady state -- an OOMKill mid-load throws + # away the compile cache this node takes ~40min to rebuild. + memory: 36Gi endpoint: port: 8000 probeOverrides: From 749165be175ea5ab6e3639171b0d96788256cf37 Mon Sep 17 00:00:00 2001 From: Tanguille <91473554+Tanguille@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:43:00 +0200 Subject: [PATCH 2/5] docs(vllm): fix comments the pool raise falsified - maxModelLen block claimed "deliberately no margin" and quoted 223,172 tokens / 1.01x; the pool is now 287,159 / 1.30x and the margin is intentional. Reframed as a decode-performance ceiling. - it also told maintainers Renovate would open the image-bump PR that triggers a re-check; Renovate cannot order a commit-tagged nightly, so that check is manual. - dshm comment restated the tier size a third time; now states the invariant and that nothing enforces it (the guard only sweeps /kvoffload). - resources.memory cited 25.1Gi, measured before the tier filled; actual is 28Gi. - engine-benchmarks-gfx1201.md still quoted 7 GiB / 221,612 tokens. --- docs/llm-hosting/engine-benchmarks-gfx1201.md | 4 +-- .../ai/llmkube/models/qwen38-27b-vllm.yaml | 34 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/llm-hosting/engine-benchmarks-gfx1201.md b/docs/llm-hosting/engine-benchmarks-gfx1201.md index 84f467aa65..654601f374 100644 --- a/docs/llm-hosting/engine-benchmarks-gfx1201.md +++ b/docs/llm-hosting/engine-benchmarks-gfx1201.md @@ -34,8 +34,8 @@ unchanged config. Measured with `node_drm_memory_vram_used_bytes` across transcode start/stop: a 4K Dolby Vision transcode costs 0.85 GB, fileflows 0.69 GB — measured separately, never concurrently, so the bound is their sum, 1.54 GB — not 4.6 GB. -Production reserves 2 GiB and sizes KV explicitly (`--kv-cache-memory 7 GiB`, 221,612 -tokens). The contention that *does* bite is compute: DV tone mapping runs on Vulkan +Production reserves 2 GiB and sizes KV explicitly (`--kv-cache-memory 9 GiB`, 287,159 +tokens; raised from 7 GiB / 223,172 on 2026-08-19, leaving 2.57 GB free at peak). The contention that *does* bite is compute: DV tone mapping runs on Vulkan shaders, so a transcode crawls at 1.15x while vLLM saturates the CUs. Throttling vLLM to fix it is not viable — `maxNumBatchedTokens: 2048` cost 4-16x TTFT on this prefill-bound workload (129:1 prompt:output). diff --git a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml index 3ad3bf0a7e..fb2cc7f7b6 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml @@ -180,22 +180,21 @@ spec: claimName: qwen38-27b-vllm-model-cache vllmConfig: # The only per-request cap on KV blocks, so it decides how much of the pool - # a single session can hold. Set to the pool so one session can use - # effectively all of it: verified booting at this value, which reports - # 223,172 tokens (raising it improves block packing slightly) = 1.01x, so - # a session reaches 99.3%. Headroom, not a working size -- Hermes peaks at + # a single session can hold. Headroom, not a working size -- Hermes peaks at # 112K and p90 prompts are 20K, and normal concurrency is unaffected # because KV is paged and allocated on demand. # - # Deliberately no margin: if a future change shrinks the pool below this - # the engine refuses to start rather than silently truncating, and Flux - # applies automatically. Re-read "GPU KV cache size" from the boot log and - # match it after ANY of: --kv-cache-memory, the mamba cache dtypes, the - # attention block size, or a vLLM image bump. That last one is the easy - # miss -- Renovate opens it as an ordinary reviewed PR. - # The pool now fits 262,144 but DO NOT raise it: 262,144 measured ~half the + # A decode-performance ceiling, NOT a pool-matching value: the pool is + # 287,159 tokens (1.30x this cap) and that margin is deliberate. DO NOT + # raise this toward the model's 262,144 limit -- 262,144 measured ~half the # single-stream decode (15.8-17.1 vs 29.7-32.0 tok/s), confirmed by exact # revert; mechanism unexplained. Production prompts are 47-56K anyway. + # + # If the pool ever shrinks below this cap the engine refuses to start + # rather than silently truncating. Re-read "GPU KV cache size" from the + # boot log after ANY of: --kv-cache-memory, the mamba cache dtypes, the + # attention block size, or a vLLM image bump -- and check by hand, since + # Renovate cannot order a commit-tagged nightly and will not open that PR. maxModelLen: 221612 kvCacheDtype: fp8_e4m3 # Confirmed working on this hybrid Mamba/GDN model (~27% hit rate under @@ -237,8 +236,9 @@ spec: - name: kv-offload persistentVolumeClaim: claimName: qwen38-27b-vllm-kv-offload - # The CPU offload tier mmaps cpu_bytes_to_use (22Gi) here; the 64M - # default EFAULTs on pre-fault. 24Gi = 22Gi + 2Gi margin. + # The CPU offload tier mmaps cpu_bytes_to_use here; the 64M default + # EFAULTs on pre-fault. Keep this at cpu_bytes_to_use + 2Gi. Nothing + # enforces that -- the kv-offload-guard CronJob only sweeps /kvoffload. - name: dshm emptyDir: medium: Memory @@ -326,10 +326,10 @@ spec: type: Unconfined resources: cpu: "2" - # Steady state was 13.3Gi with an 8Gi shm offload tier (tmpfs, so charged - # here); the 22Gi tier measured 25.1Gi total. Headroom kept for the load - # spike rather than trimmed to steady state -- an OOMKill mid-load throws - # away the compile cache this node takes ~40min to rebuild. + # tmpfs is charged here, so this tracks cpu_bytes_to_use: measured 28Gi + # total with the tier full. Headroom kept for the load spike rather than + # trimmed -- an OOMKill mid-load throws away the compile cache this node + # takes ~40min to rebuild. memory: 36Gi endpoint: port: 8000 From a2197dd24a40f5fe03304bfcf9644cf75f50073b Mon Sep 17 00:00:00 2001 From: Tanguille <91473554+Tanguille@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:55:27 +0200 Subject: [PATCH 3/5] docs(vllm): rollback criteria for the CPU tier, exact peak memory - state when to roll the 22Gi tier back to 16Gi (24h restart-free soak that fails to beat the 0.5333 external-hit baseline, or node memory pressure), per review feedback that an unvalidated change needs one - replace the author-reported "28Gi total" with the measured 28.3Gi peak under 4x32K concurrent load with the tier full, 7.7Gi headroom --- kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml index fb2cc7f7b6..247e0df12a 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml @@ -310,6 +310,10 @@ spec: # CPU tier 22Gi: the 16Gi tier cascaded 3.5 GB/hour to the fs tier, i.e. # eviction pressure. Benefit NOT yet validated -- needs a restart-free # window, since this tier is an emptyDir tmpfs that every restart wipes. + # Roll back to 17179869184 (16Gi, dshm 18Gi, memory 32Gi) if after a 24h + # restart-free soak vllm:external_prefix_cache_hits_total/queries_total has + # not beaten its 0.5333 pre-change baseline, or if control-1 shows memory + # pressure or OOMKills. # DO NOT remove the fs secondary tier to save Ceph writes: its 4.5% hit # rate is not its job. Removing it halved single-stream decode (15.5 vs # 31 tok/s, exact-revert confirmed) -- it keeps eviction asynchronous. @@ -326,8 +330,8 @@ spec: type: Unconfined resources: cpu: "2" - # tmpfs is charged here, so this tracks cpu_bytes_to_use: measured 28Gi - # total with the tier full. Headroom kept for the load spike rather than + # tmpfs is charged here, so this tracks cpu_bytes_to_use: 28.3Gi peak + # under 4x32K concurrent load with the tier full (7.7Gi headroom). Headroom kept for the load spike rather than # trimmed -- an OOMKill mid-load throws away the compile cache this node # takes ~40min to rebuild. memory: 36Gi From cdfa05ea6acd8837b5f0642c9016646d5fc527c1 Mon Sep 17 00:00:00 2001 From: Tanguille <91473554+Tanguille@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:00:23 +0200 Subject: [PATCH 4/5] style(vllm): rewrap the memory comment to the file's line width --- kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml index 247e0df12a..b0ebb06db9 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml @@ -331,9 +331,10 @@ spec: resources: cpu: "2" # tmpfs is charged here, so this tracks cpu_bytes_to_use: 28.3Gi peak - # under 4x32K concurrent load with the tier full (7.7Gi headroom). Headroom kept for the load spike rather than - # trimmed -- an OOMKill mid-load throws away the compile cache this node - # takes ~40min to rebuild. + # under 4x32K concurrent load with the tier full (7.7Gi headroom). + # Headroom kept for the load spike rather than trimmed -- an OOMKill + # mid-load throws away the compile cache this node takes ~40min to + # rebuild. memory: 36Gi endpoint: port: 8000 From 4c1f1910af04997116de3287c38f9f3fcf872d37 Mon Sep 17 00:00:00 2001 From: Tanguille <91473554+Tanguille@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:06:55 +0200 Subject: [PATCH 5/5] fix(renovate): let Renovate track vllm nightly builds The commit-tagged `nightly-` pin is immutable, so the docker datasource has no ordering scheme and never detects a new build -- vLLM nightlies were invisible to Renovate entirely. Pin the rolling `nightly` tag by digest instead (same shape as kubesearch-mcp:master@sha256:). Identical image today: both tags currently resolve to sha256:5ea7099b. Renovate can now re-resolve the tag and raise a digest-bump PR. Added a packageRule to keep those bumps out of auto-merge -- one rolls the single-GPU serving pod and can move the KV pool maxModelLen is sized against. --- .renovaterc.json5 | 6 ++++++ .../apps/ai/llmkube/models/qwen38-27b-vllm.yaml | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.renovaterc.json5 b/.renovaterc.json5 index e7212be493..61118a4c83 100644 --- a/.renovaterc.json5 +++ b/.renovaterc.json5 @@ -75,6 +75,12 @@ automerge: true, automergeType: "pr" }, + { + description: "vllm-openai-rocm nightly digest — the rolling `nightly` tag is digest-pinned so Renovate can detect new builds at all; never auto-merge, a bump rolls the single-GPU serving pod and can move the KV pool that maxModelLen is sized against", + matchDatasources: ["docker"], + matchPackageNames: ["vllm/vllm-openai-rocm"], + automerge: false + }, // Grouping rules { description: "Actions Runner Controller Group", diff --git a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml index b0ebb06db9..0f1c6a7f4a 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml @@ -173,9 +173,13 @@ spec: # translates this to --max-num-seqs for the vllm runtime. parallelSlots: 16 bindAddress: 0.0.0.0 - # Commit-tagged nightly, not the rolling `nightly` tag. + # Rolling `nightly` tag pinned by digest, same shape as + # kubesearch-mcp:master@sha256:. The digest is what gets pulled, so this is + # as immutable as a commit tag -- but Renovate can re-resolve the tag and + # open a digest-bump PR, which it can never do for an immutable + # nightly- tag (no ordering scheme, so no update is ever detected). # v0.27.1 is the newest release and predates gfx1201 support. - image: vllm/vllm-openai-rocm:nightly-5a4c8d99242e9e069b604d0e9b969e77f7dd501d@sha256:5ea7099b53cdf9049ddcd446f15d831f32bce06c2822d6f8f228b819f189fc48 + image: vllm/vllm-openai-rocm:nightly@sha256:5ea7099b53cdf9049ddcd446f15d831f32bce06c2822d6f8f228b819f189fc48 modelCache: claimName: qwen38-27b-vllm-model-cache vllmConfig: @@ -193,8 +197,8 @@ spec: # If the pool ever shrinks below this cap the engine refuses to start # rather than silently truncating. Re-read "GPU KV cache size" from the # boot log after ANY of: --kv-cache-memory, the mamba cache dtypes, the - # attention block size, or a vLLM image bump -- and check by hand, since - # Renovate cannot order a commit-tagged nightly and will not open that PR. + # attention block size, or a vLLM image bump -- Renovate raises that last + # one as a digest-bump PR, so re-check this value when reviewing it. maxModelLen: 221612 kvCacheDtype: fp8_e4m3 # Confirmed working on this hybrid Mamba/GDN model (~27% hit rate under