diff --git a/.github/workflows/agent-pr-review.yaml b/.github/workflows/agent-pr-review.yaml index d08f6e094f..34b7d05168 100644 --- a/.github/workflows/agent-pr-review.yaml +++ b/.github/workflows/agent-pr-review.yaml @@ -43,22 +43,22 @@ jobs: ai_base_url: http://litellm.ai.svc.cluster.local/v1 ai_api_key: ${{ secrets.LITELLM_API_KEY }} # omniroute routes to a larger free-tier model than the self-hosted 27B - # (see litellm routerSettings.fallbacks); falls back to qwen-3.6-fast + # (see litellm routerSettings.fallbacks); falls back to qwen-3.8-fast # automatically if the free provider fails or times out. ai_model: omniroute # v2.1.10 inherits ai_fallback_base_url from ai_base_url when unset, then # requires ai_fallback_model whenever ai_fallback_base_url is set (even # inherited) — fails fast with no model call otherwise. Point this at - # qwen-3.6-fast directly (same litellm endpoint) rather than duplicating + # qwen-3.8-fast directly (same litellm endpoint) rather than duplicating # ai_model: litellm's routerSettings.fallbacks already retries omniroute # failures internally, so this only fires if that whole chain fails — # a real second attempt via a distinct model, not a no-op retry. - ai_fallback_model: qwen-3.6-fast - # omniroute's proxied models don't share SGLang's sampling_defaults, so + ai_fallback_model: qwen-3.8-fast + # omniroute's proxied models don't share the self-hosted model's defaults, so # an empty temperature has no known-good fallback here — pin explicitly. ai_temperature: "0.2" ai_response_format: json_schema - # Covers both omniroute (observed 2-74s) and, on fallback, qwen-3.6-fast's + # Covers both omniroute (observed 2-74s) and, on fallback, qwen-3.8-fast's # ~10 tok/s cold prefill on this GPU (155-220s near ~100K ctx, validated). ai_request_timeout_sec: "600" verdict_policy: findings_severity_gated diff --git a/.renovaterc.json5 b/.renovaterc.json5 index 70f60ca0ef..e7212be493 100644 --- a/.renovaterc.json5 +++ b/.renovaterc.json5 @@ -42,8 +42,12 @@ description: "HuggingFace model files revision-pinned in llmkube Model sources (HF model repos are git repos; digest = repo commit sha)", customType: "regex", managerFilePatterns: ["/kubernetes/apps/ai/llmkube/models/.+\\.ya?ml$/"], + // Two source forms in use: the resolve/ URL for single-file GGUFs, and + // hf://repo@sha for multi-file repos (qwen38, muse-glimmer). Without the + // second, the primary serving model's weights digest tracks nothing. matchStrings: [ - "source:\\s+https://huggingface\\.co/(?[^/\\s]+/[^/\\s]+)/resolve/(?[a-f0-9]{40})/" + "source:\\s+https://huggingface\\.co/(?[^/\\s]+/[^/\\s]+)/resolve/(?[a-f0-9]{40})/", + "source:\\s+hf://(?[^/\\s]+/[^/\\s]+)@(?[a-f0-9]{40})" ], packageNameTemplate: "https://huggingface.co/{{depName}}", currentValueTemplate: "main", @@ -52,7 +56,9 @@ { description: "opencode npm plugins (executed in-process in an agent pod, so they stay pinned)", customType: "regex", - managerFilePatterns: ["/kubernetes/apps/ai/opencode/app/config/opencode\\.jsonc$/"], + managerFilePatterns: [ + "/kubernetes/apps/ai/opencode/app/config/opencode\\.jsonc$/" + ], matchStrings: [ '"(?@?[\\w.-]+(?:/[\\w.-]+)?)@(?\\d+\\.\\d+\\.\\d+)"' ], @@ -70,12 +76,6 @@ automergeType: "pr" }, // Grouping rules - { - description: "sglang-rdna4 deployed image digest — built by our fork of the upstream RDNA4 image, not a pipeline in this repo; review before the HelmRelease rolls the serving pod (Recreate on the single GPU)", - matchDatasources: ["docker"], - matchPackageNames: ["ghcr.io/tanguille/sglang-rdna4"], - automerge: false - }, { description: "Actions Runner Controller Group", groupName: "actions-runner-controller", @@ -99,7 +99,7 @@ matchDatasources: ["docker", "github-releases"], // matches all siderolabs/* packages (talos, installer, etc.); both slashes are regex // delimiters, so the inner one must stay escaped AND closed or config validation fails - matchPackageNames: ["/siderolabs\\//"], + matchPackageNames: ["/siderolabs\\//"] }, { description: "Rook-Ceph Group", diff --git a/docs/llm-hosting/bench/concsweep.py b/docs/llm-hosting/bench/concsweep.py index 61b81dfa17..5a03d9ff7e 100644 --- a/docs/llm-hosting/bench/concsweep.py +++ b/docs/llm-hosting/bench/concsweep.py @@ -17,7 +17,9 @@ def _port(): URL = f"http://127.0.0.1:{_port()}/v1/completions" -MODEL = "qwen-3.6" +# 2nd arg overrides the served-model-name, since the port-forward target decides +# which engine answers. Hardcoding it once cost a whole run of silent 0.00 tok/s. +MODEL = sys.argv[2] if len(sys.argv) > 2 else "qwen-3.8" GEN = 96 diff --git a/docs/llm-hosting/bench/spectest.py b/docs/llm-hosting/bench/spectest.py index 82949545af..d63ff1d396 100644 --- a/docs/llm-hosting/bench/spectest.py +++ b/docs/llm-hosting/bench/spectest.py @@ -14,6 +14,9 @@ def _port(): URL = f"http://127.0.0.1:{_port()}/v1/completions" +# 2nd arg overrides the served-model-name, since the port-forward target decides +# which engine answers. Hardcoding it once cost a whole run of silent 0.00 tok/s. +MODEL = sys.argv[2] if len(sys.argv) > 2 else "qwen-3.8" CODE = "\n".join(f"def handler_{i}(request, context):\n" f" payload = request.get('payload_{i}')\n" f" if payload is None:\n" @@ -21,7 +24,7 @@ def _port(): f" return {{'status': 200, 'body': payload}}\n" for i in range(240)) prompt = (f"Here is a Python module:\n\n{CODE}\n\n" "Reproduce handler_0 through handler_12 EXACTLY as written above, verbatim:\n\n") -b = json.dumps({"model": "qwen-3.6", "prompt": prompt, "max_tokens": 400, +b = json.dumps({"model": MODEL, "prompt": prompt, "max_tokens": 400, "temperature": 0, "ignore_eos": True}).encode() t0 = time.perf_counter() rq = urllib.request.Request(URL, data=b, headers={"Content-Type": "application/json"}) diff --git a/docs/llm-hosting/engine-benchmarks-gfx1201.md b/docs/llm-hosting/engine-benchmarks-gfx1201.md index b6d89fa6f1..126819dff4 100644 --- a/docs/llm-hosting/engine-benchmarks-gfx1201.md +++ b/docs/llm-hosting/engine-benchmarks-gfx1201.md @@ -4,10 +4,10 @@ Multi-engine measurement record for Qwen3.6-27B on RDNA4 — vLLM, SGLang and ll Measurements are dated and kept as a historical series; the open experiments at the bottom are still outstanding. -**Current production engine: SGLang**, `ghcr.io/tanguille/sglang-rdna4`, built by our fork -of the upstream RDNA4 image rather than a pipeline in this repo — see -`docs/llm-hosting/sglang-blockers.md` "Current approach" for the build source and the -upstream cutover plan; outstanding upstream gaps are tracked in the same doc. The +**Current production engine: vLLM**, serving Qwen 3.8 as `qwen38-27b-vllm` (2026-08-16). +SGLang and the whole Qwen 3.6 lane were retired with that cutover; +`docs/llm-hosting/sglang-blockers.md` is kept as the record of why SGLang never +displaced vLLM here. The 2026-06-21 round below concluded in favour of vLLM and has since been superseded — read its numbers as a snapshot of that date, not as current guidance. diff --git a/docs/llm-hosting/sglang-blockers.md b/docs/llm-hosting/sglang-blockers.md index 9252ca3a7a..2c78dcccd9 100644 --- a/docs/llm-hosting/sglang-blockers.md +++ b/docs/llm-hosting/sglang-blockers.md @@ -2,7 +2,7 @@ Tracking what needs to land upstream before SGLang can replace vLLM in production without depending on the mattbucci RDNA4 fork. -**Current approach:** running `ghcr.io/tanguille/sglang-rdna4:sha-cb7b7605...` (torch 2.11+rocm7.2, Triton 3.6, SGLang v0.5.16), built by our fork's own `build-image.yaml` `publish` job (`Tanguille/2x-R9700-RDNA4-GFX1201-sglang-inference`, `main`), not by a pipeline in this repo — `docker/sglang-rdna4/` and `.github/workflows/build-sglang-rdna4.yaml` are retired. That fork branch carries mattbucci's full patch series plus our HiCache OpenSSL-headers fix (`fix/hicache-openssl-headers`, upstream PR mattbucci#6, still open/unmerged there — this is our own fork build, not the upstream package). `SGLANG_RDNA4_DISABLE_STORE_CACHE=1` is required in the HelmRelease env because LLMKube bypasses the image's entrypoint (see the InferenceService manifest). Once mattbucci#6 merges upstream, re-point at `ghcr.io/mattbucci/sglang-rdna4` directly and drop the fork build. The retired PVC-rebuild recipe is gone with the `sglang` app directory; recover it from git history if ever needed. +**Historical approach (retired 2026-08-16, superseded by vLLM/Qwen 3.8):** ran `ghcr.io/tanguille/sglang-rdna4:sha-cb7b7605...` (torch 2.11+rocm7.2, Triton 3.6, SGLang v0.5.16), built by our fork's own `build-image.yaml` `publish` job (`Tanguille/2x-R9700-RDNA4-GFX1201-sglang-inference`, `main`), not by a pipeline in this repo — `docker/sglang-rdna4/` and `.github/workflows/build-sglang-rdna4.yaml` are retired. That fork branch carries mattbucci's full patch series plus our HiCache OpenSSL-headers fix (`fix/hicache-openssl-headers`, upstream PR mattbucci#6, still open/unmerged there — this is our own fork build, not the upstream package). `SGLANG_RDNA4_DISABLE_STORE_CACHE=1` was required in the HelmRelease env because LLMKube bypasses the image's entrypoint (see the InferenceService manifest). Once mattbucci#6 merges upstream, re-point at `ghcr.io/mattbucci/sglang-rdna4` directly and drop the fork build. The retired PVC-rebuild recipe is gone with the `sglang` app directory; recover it from git history if ever needed. **Fork state check (2026-08-01):** the deployed image `sha-cb7b76050cbf` already sits on the fork's v0.5.16 rebase (`689339d`, 2026-07-27), so blocker 6's #31648 (mamba LRU) is live in production — it ships in the v0.5.16 release. Fork patches 086 (AMD Triton `num_kv_splits` 16→64, claimed 2.14× @256K) and 087 (bf16 page-vector attention, claimed +21% @256K) are also in that image; both are model-agnostic (verified via patch content) but were only A/B'd on `coder-reap-25b`, not Qwen3.6-27B. A clean TP=1 re-bench of the deployed image is still outstanding (the 2026-08-01 attempt ran under production load and is recorded as a floor in `engine-benchmarks-gfx1201.md`). `diff cb7b760..origin/main` is docs/eval-only, so a rebuild on current fork main gains no runtime changes today. diff --git a/docs/llm-hosting/vllm-vs-sglang-2026-07.md b/docs/llm-hosting/vllm-vs-sglang-2026-07.md index 3fd498a9a7..a0c775420e 100644 --- a/docs/llm-hosting/vllm-vs-sglang-2026-07.md +++ b/docs/llm-hosting/vllm-vs-sglang-2026-07.md @@ -236,9 +236,10 @@ Both drive the OpenAI `/v1/completions` endpoint, so they run unmodified against engine. Port-forward the service and pass the port: ```sh -kubectl -n ai port-forward svc/qwen36-27b 30000:30000 & -python3 concsweep.py 30000 -python3 spectest.py 30000 +kubectl -n ai port-forward svc/qwen38-27b-vllm 8000:8000 & +python3 concsweep.py 8000 +python3 spectest.py 8000 +# optional 2nd arg overrides the served-model-name (default qwen-3.8) ``` **Discard the first run after a pod restart** (see above), and note `concsweep.py` reports diff --git a/kubernetes/apps/ai/litellm/instance/dashboard/litellm.json b/kubernetes/apps/ai/litellm/instance/dashboard/litellm.json deleted file mode 100644 index e2e23af1e5..0000000000 --- a/kubernetes/apps/ai/litellm/instance/dashboard/litellm.json +++ /dev/null @@ -1,550 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "description": "LiteLLM proxy metrics. grafana.com 24965 is broken here: job/instance vars use litellm_proxy_total_requests_metric_created, which is not exported (only *_total exists).", - "editable": true, - "graphTooltip": 1, - "links": [ - { - "icon": "external link", - "includeVars": true, - "keepTime": true, - "title": "SGLang", - "type": "link", - "url": "/d/sglang/sglang" - }, - { - "icon": "external link", - "includeVars": true, - "keepTime": true, - "title": "LLMKube", - "type": "link", - "url": "/d/llmkube-inference/llmkube-inference" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 1, - "title": "Overview", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "0": { "color": "red", "index": 0, "text": "DOWN" }, - "1": { "color": "green", "index": 1, "text": "UP" } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - } - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "id": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "min(up{job=~\"$job\", instance=~\"$instance\"})", - "instant": true, - "refId": "A" - } - ], - "title": "Scrape Up", - "type": "stat" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { "unit": "reqps", "decimals": 3 }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 5, "x": 4, "y": 1 }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(litellm_proxy_total_requests_metric_total{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\", status_code=\"200\"}[$__rate_interval]))", - "instant": true, - "refId": "A" - } - ], - "title": "Success req/s", - "type": "stat" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.001 }, - { "color": "red", "value": 0.01 } - ] - }, - "unit": "reqps", - "decimals": 3 - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 5, "x": 9, "y": 1 }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(litellm_proxy_total_requests_metric_total{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\", status_code!=\"200\", status_code!=\"None\"}[$__rate_interval]))", - "instant": true, - "refId": "A" - } - ], - "title": "Error req/s", - "type": "stat" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 5, "x": 14, "y": 1 }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(litellm_in_flight_requests{job=~\"$job\", instance=~\"$instance\"})", - "instant": true, - "refId": "A" - } - ], - "title": "In-flight", - "type": "stat" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(litellm_proxy_failed_requests_metric_total{job=~\"$job\", instance=~\"$instance\"})", - "instant": true, - "refId": "A" - } - ], - "title": "Failed (total)", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "id": 10, - "title": "Traffic", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "showPoints": "never", - "spanNulls": true - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "id": 11, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (requested_model, route, status_code) (rate(litellm_proxy_total_requests_metric_total{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval]))", - "legendFormat": "{{requested_model}} {{route}} {{status_code}}", - "refId": "A" - } - ], - "title": "Request rate by model / route", - "type": "timeseries" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "showPoints": "never", - "spanNulls": true - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "id": 12, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (requested_model) (rate(litellm_input_tokens_metric_total{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval]))", - "legendFormat": "{{requested_model}} in", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (requested_model) (rate(litellm_output_tokens_metric_total{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval]))", - "legendFormat": "{{requested_model}} out", - "refId": "B" - } - ], - "title": "Token rate", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, - "id": 20, - "title": "Latency", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "showPoints": "never", - "spanNulls": true - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, - "id": 21, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, sum by (requested_model, le) (rate(litellm_llm_api_latency_metric_bucket{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval])))", - "legendFormat": "{{requested_model}} p50", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.9, sum by (requested_model, le) (rate(litellm_llm_api_latency_metric_bucket{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval])))", - "legendFormat": "{{requested_model}} p90", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, sum by (requested_model, le) (rate(litellm_llm_api_latency_metric_bucket{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval])))", - "legendFormat": "{{requested_model}} p99", - "refId": "C" - } - ], - "title": "LLM API latency", - "type": "timeseries" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "showPoints": "never", - "spanNulls": true - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, - "id": 22, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, sum by (requested_model, le) (rate(litellm_llm_api_time_to_first_token_metric_bucket{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval])))", - "legendFormat": "{{requested_model}} p50", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.9, sum by (requested_model, le) (rate(litellm_llm_api_time_to_first_token_metric_bucket{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval])))", - "legendFormat": "{{requested_model}} p90", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, sum by (requested_model, le) (rate(litellm_llm_api_time_to_first_token_metric_bucket{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval])))", - "legendFormat": "{{requested_model}} p99", - "refId": "C" - } - ], - "title": "Time to first token", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, - "id": 30, - "title": "Failures", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "showPoints": "never", - "spanNulls": true - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, - "id": 31, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (requested_model) (rate(litellm_llm_api_failed_requests_metric_total{job=~\"$job\", instance=~\"$instance\", requested_model=~\"$model\"}[$__rate_interval]))", - "legendFormat": "{{requested_model}}", - "refId": "A" - } - ], - "title": "LLM API failures / s", - "type": "timeseries" - }, - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "showPoints": "never", - "spanNulls": true - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, - "id": 32, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (litellm_model_name) (rate(litellm_deployment_failure_responses_total{job=~\"$job\", instance=~\"$instance\", litellm_model_name=~\"$model\"}[$__rate_interval]))", - "legendFormat": "{{litellm_model_name}}", - "refId": "A" - } - ], - "title": "Deployment failures / s", - "type": "timeseries" - } - ], - "refresh": "1m", - "schemaVersion": 41, - "tags": ["ai", "litellm"], - "templating": { - "list": [ - { - "current": {}, - "hide": 0, - "label": "Datasource", - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "type": "datasource" - }, - { - "current": { "selected": true, "text": "litellm", "value": "litellm" }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(litellm_proxy_total_requests_metric_total, job)", - "includeAll": false, - "label": "Job", - "name": "job", - "query": { - "query": "label_values(litellm_proxy_total_requests_metric_total, job)", - "refId": "A" - }, - "refresh": 2, - "type": "query" - }, - { - "allValue": ".*", - "current": { "selected": true, "text": "All", "value": "$__all" }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(litellm_proxy_total_requests_metric_total{job=~\"$job\"}, instance)", - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "query": { - "query": "label_values(litellm_proxy_total_requests_metric_total{job=~\"$job\"}, instance)", - "refId": "A" - }, - "refresh": 2, - "type": "query" - }, - { - "allValue": ".*", - "current": { "selected": true, "text": "All", "value": "$__all" }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(litellm_proxy_total_requests_metric_total{job=~\"$job\", instance=~\"$instance\"}, requested_model)", - "includeAll": true, - "label": "Model", - "multi": true, - "name": "model", - "query": { - "query": "label_values(litellm_proxy_total_requests_metric_total{job=~\"$job\", instance=~\"$instance\"}, requested_model)", - "refId": "A" - }, - "refresh": 2, - "type": "query" - } - ] - }, - "time": { "from": "now-6h", "to": "now" }, - "timezone": "browser", - "title": "LiteLLM", - "uid": "ad8llmv", - "version": 1 -} diff --git a/kubernetes/apps/ai/litellm/instance/grafanadashboard.yaml b/kubernetes/apps/ai/litellm/instance/grafanadashboard.yaml index cc82526720..c0f5cf5bd1 100644 --- a/kubernetes/apps/ai/litellm/instance/grafanadashboard.yaml +++ b/kubernetes/apps/ai/litellm/instance/grafanadashboard.yaml @@ -12,8 +12,8 @@ spec: datasources: - datasourceName: VictoriaMetrics inputName: DS_PROMETHEUS - # grafana.com 24965 job/instance vars use litellm_proxy_total_requests_metric_created - # (not exported); leaves $job empty and every panel blank. - configMapRef: - name: litellm-dashboard - key: litellm.json + # Was a vendored copy: 24965's $job/$instance vars need + # litellm_proxy_total_requests_metric_created, unexported until v1.97.0. + # No revision -- track latest. + grafanaCom: + id: 24965 diff --git a/kubernetes/apps/ai/litellm/instance/kustomization.yaml b/kubernetes/apps/ai/litellm/instance/kustomization.yaml index 7f26aaa074..42277276bc 100644 --- a/kubernetes/apps/ai/litellm/instance/kustomization.yaml +++ b/kubernetes/apps/ai/litellm/instance/kustomization.yaml @@ -9,13 +9,3 @@ resources: - models.yaml - servicemonitor.yaml - grafanadashboard.yaml - -configMapGenerator: - - name: litellm-dashboard - files: - - dashboard/litellm.json - options: - annotations: - kustomize.toolkit.fluxcd.io/substitute: disabled -generatorOptions: - disableNameSuffixHash: true diff --git a/kubernetes/apps/ai/litellm/instance/models.yaml b/kubernetes/apps/ai/litellm/instance/models.yaml index 09fbe9d578..17d2a64262 100644 --- a/kubernetes/apps/ai/litellm/instance/models.yaml +++ b/kubernetes/apps/ai/litellm/instance/models.yaml @@ -3,46 +3,55 @@ apiVersion: litellm.home-operations.com/v1alpha1 kind: LiteLLMModel metadata: - name: qwen-3.6 + name: qwen-3.8 spec: - modelName: qwen-3.6 + modelName: qwen-3.8 proxyRef: litellm params: - model: openai/qwen-3.6 - apiBase: http://qwen36-27b.ai.svc.cluster.local:30000/v1 - # Native vLLM/SGLang are unauthenticated internally, but LiteLLM's OpenAI provider + # Renamed from qwen-3.6 together with the consumers that hardcode it: + # karakeep, agent-pr-review.yaml, proxy.yaml fallbacks. NOT hermes -- its + # profiles live on the hermes data PVC, so they are edited by hand and no + # rollback of this repo touches them. + # Backend swapped to qwen38-27b-vllm 2026-08-16 -- numbers in that file. + model: openai/qwen-3.8 + apiBase: http://qwen38-27b-vllm.ai.svc.cluster.local:8000/v1 + # Native vLLM is unauthenticated internally, but LiteLLM's OpenAI provider # requires a non-empty api_key. - apiKey: sk-sglang-noauth + apiKey: sk-vllm-noauth additional: timeout: 1800 stream_timeout: 1800 num_retries: 0 info: mode: chat - # SGLang's 180000 ctx minus maxOutputTokens. - maxInputTokens: 171808 + # vLLM's maxModelLen (112000) minus maxOutputTokens; covers Hermes' measured + # p100 (112K), so routine sessions never truncate. Takes both the + # OffloadingConnector and an explicit --kv-cache-memory to reach — see + # qwen38-27b-vllm.yaml. + maxInputTokens: 103808 maxOutputTokens: 8192 --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json apiVersion: litellm.home-operations.com/v1alpha1 kind: LiteLLMModel metadata: - name: qwen-3.6-fast + name: qwen-3.8-fast spec: - modelName: qwen-3.6-fast + modelName: qwen-3.8-fast proxyRef: litellm params: - model: openai/qwen-3.6 - apiBase: http://qwen36-27b.ai.svc.cluster.local:30000/v1 - apiKey: sk-sglang-noauth + model: openai/qwen-3.8 + apiBase: http://qwen38-27b-vllm.ai.svc.cluster.local:8000/v1 + apiKey: sk-vllm-noauth additional: # Hermes compression is a ~100K+ prefill on this alias; under GPU contention # the shared prefill rate drops below what 600s covers (2026-07-07). timeout: 900 stream_timeout: 900 num_retries: 0 - # Vendor non-thinking sampling (Qwen3.6 model card) — generation_config ships thinking-mode - # values (1.0/0.95/presence 0). Defaults only: client-set params win (v1.91.0 router merge). + # Non-thinking instruct-mode sampling — matches Qwen3.8's model card + # exactly (temperature/top_p/presence_penalty), same as Qwen3.6's. + # Defaults only: client-set params win (v1.91.0 router merge). temperature: 0.7 top_p: 0.8 presence_penalty: 1.5 @@ -51,7 +60,8 @@ spec: enable_thinking: false info: mode: chat - maxInputTokens: 171808 + # 112000 - 8192, as above. + maxInputTokens: 103808 maxOutputTokens: 8192 --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json @@ -68,10 +78,10 @@ spec: model: openai/auto/smart apiBase: http://omniroute.ai.svc.cluster.local:20128/v1 # REQUIRE_API_KEY=false internally; LiteLLM's OpenAI provider still needs - # a non-empty api_key (same convention as the qwen-3.6 sk-sglang-noauth). + # a non-empty api_key (same convention as qwen-3.8's sk-vllm-noauth). apiKey: sk-omniroute-noauth additional: - # Bounded so the qwen-3.6-fast fallback fires inside the pr-reviewer-action's + # Bounded so the qwen-3.8-fast fallback fires inside the pr-reviewer-action's # 600s curl timeout instead of the action giving up and retrying blind # (observed 2026-08-03: 60min run, fallback never triggered). Timeouts # aren't in the proxy's retry_policy and num_retries is 0, so one attempt diff --git a/kubernetes/apps/ai/litellm/instance/proxy.yaml b/kubernetes/apps/ai/litellm/instance/proxy.yaml index 9b026537a8..923d618afd 100644 --- a/kubernetes/apps/ai/litellm/instance/proxy.yaml +++ b/kubernetes/apps/ai/litellm/instance/proxy.yaml @@ -49,17 +49,17 @@ spec: drop_params: true num_retries: 0 routerSettings: - # Per-error override of num_retries: 0 — a dropped SGLang connection costs a + # Per-error override of num_retries: 0 — a dropped backend connection costs a # whole Hermes cron run. Timeouts stay at 0: re-running a ~100K prefill that # already burned 900s only deepens the contention that caused it. retry_policy: InternalServerErrorRetries: 2 # omniroute's free providers have no SLA and can fail/time out without # warning; fall back to the self-hosted (slower but always-available) - # qwen-3.6-fast alias so a flaky free provider doesn't take out the PR + # qwen-3.8-fast alias so a flaky free provider doesn't take out the PR # review action. fallbacks: - - omniroute: ["qwen-3.6-fast"] + - omniroute: ["qwen-3.8-fast"] route: hostnames: - "litellm.${SECRET_DOMAIN}" diff --git a/kubernetes/apps/ai/llmkube/app/helmrelease.yaml b/kubernetes/apps/ai/llmkube/app/helmrelease.yaml index 560a2554a4..01436b8751 100644 --- a/kubernetes/apps/ai/llmkube/app/helmrelease.yaml +++ b/kubernetes/apps/ai/llmkube/app/helmrelease.yaml @@ -41,10 +41,11 @@ spec: dashboards: enabled: true namespace: observability - # The rest render blank here: amd-gpu-observability queries amdgpu_* - # (we run drm-exporter), llmkube-quota and model-router need features we - # don't deploy, llmkube-slo needs pyrra burn-rate rules. - only: [llmkube-inference, sglang-dashboard] + # No `only`: ship everything and let the operator's default auto mode + # gate the runtime dashboards on a live InferenceService. Auto covers + # sglang/vllm/llamacpp only, so amd-gpu-observability, llmkube-quota, + # llmkube-slo and model-router ship unconditionally and render blank + # (no pyrra, and we export drm_* not amdgpu_*). operator: enabled: true folder: ai @@ -58,7 +59,7 @@ spec: # Shared RWX cache on CephFS: the controller and the iGPU InferenceServices # land on different nodes, so an RWO volume multi-attach-errors. Access is # write-once on download then read-once at load, so CephFS latency never - # touches inference. qwen36-27b brings its own node-local modelCache. + # touches inference. qwen38-27b-vllm brings its own node-local modelCache. # The model-cache-prep init chowns /models as root, so CephFS (fsGroupPolicy: # None) works too. modelCache: diff --git a/kubernetes/apps/ai/llmkube/models/kustomization.yaml b/kubernetes/apps/ai/llmkube/models/kustomization.yaml index 75802d4772..778ea9dc8d 100644 --- a/kubernetes/apps/ai/llmkube/models/kustomization.yaml +++ b/kubernetes/apps/ai/llmkube/models/kustomization.yaml @@ -3,9 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - qwen36-27b-sglang.yaml + - qwen38-27b-vllm.yaml - qwen3-embedding.yaml - qwen35-2b.yaml - vmcp-embedding.yaml - - qwen36-27b-vllm.yaml - muse-glimmer-30b.yaml diff --git a/kubernetes/apps/ai/llmkube/models/muse-glimmer-30b.yaml b/kubernetes/apps/ai/llmkube/models/muse-glimmer-30b.yaml index 63c98f65c6..39fe901265 100644 --- a/kubernetes/apps/ai/llmkube/models/muse-glimmer-30b.yaml +++ b/kubernetes/apps/ai/llmkube/models/muse-glimmer-30b.yaml @@ -48,7 +48,7 @@ spec: modelRef: muse-glimmer-30b modelCache: claimName: muse-glimmer-30b-model-cache - # Config is evaluated and kept; scale to 1 only with qwen36-27b-vllm at 0. + # Config is evaluated and kept; scale to 1 only with qwen38-27b-vllm at 0. replicas: 0 # Needs >= b10361 for muse_glimmer. image: ghcr.io/ggml-org/llama.cpp:server-rocm-b10454@sha256:799571945f574c640d1151633f49c202cd778bf5a227df6ef314a1f08623f5d3 diff --git a/kubernetes/apps/ai/llmkube/models/qwen3-embedding.yaml b/kubernetes/apps/ai/llmkube/models/qwen3-embedding.yaml index 7f741b2055..3d9b74da58 100644 --- a/kubernetes/apps/ai/llmkube/models/qwen3-embedding.yaml +++ b/kubernetes/apps/ai/llmkube/models/qwen3-embedding.yaml @@ -59,7 +59,8 @@ spec: - "0" # Pin to the iGPU node; without this the scheduler could place it on the dGPU - # node (which also advertises squat.ai/dri) and steal a dGPU slot from sglang. + # node (which also advertises squat.ai/dri) and steal a dGPU slot from the + # 27B serving pod. nodeSelector: amd.com/igpu: "true" diff --git a/kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml b/kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml deleted file mode 100644 index e95540e9b0..0000000000 --- a/kubernetes/apps/ai/llmkube/models/qwen36-27b-sglang.yaml +++ /dev/null @@ -1,336 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: qwen36-27b-model-cache - namespace: ai -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath - resources: - requests: - storage: 50Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: qwen36-27b-triton-cache - namespace: ai -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath - resources: - requests: - storage: 10Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: qwen36-27b-hicache - namespace: ai -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath - resources: - # hostpath enforces no quota, so the real bound is the evictor's MAX_SIZE below. - requests: - storage: 64Gi ---- -# Prunes HiCache generations superseded by an image bump (why they are versioned at all: -# see SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR below). This CRD has no initContainers field -# to automate it, so it was manual, and one missed cleanup orphaned 64G on 2026-07-30 -- -# against 60.3G free on control-1's sparse TrueNAS zvol, where filling the pool stops the -# VM. Tradeoff: rolling back >1 day after a bump finds that cache pruned (cold TTFT). -apiVersion: batch/v1 -kind: CronJob -metadata: - name: qwen36-27b-hicache-prune - namespace: ai -spec: - schedule: "30 4 * * *" - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 2 - template: - spec: - restartPolicy: OnFailure - securityContext: - # Superseded generations were written by root-era pods (pre-4a4b06f). - runAsUser: 0 - containers: - - name: prune - image: ghcr.io/home-operations/busybox:1.38.0@sha256:7e2c04dd50ede647bf4a7a4c8dbd629dd4971cd139b9b88fb22bfc3c7a6c13df - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: - - /bin/sh - - -c - - | - set -eu - cd /hicache - # Newest mtime = the live generation; a serving pod keeps promoting into - # it. Skip it and anything written in the last day, so a draining pod - # keeps its cache through a rollout. - newest="$(ls -1dt -- */ 2>/dev/null | head -1)" - for d in */; do - [ -d "$d" ] || continue - [ "$d" = "$newest" ] && continue - [ -n "$(find "$d" -maxdepth 0 -mmin +1440)" ] || continue - echo "pruning superseded generation: $d" - rm -rf -- "$d" - done - # Names/counts only: du here would stat the whole store (682,508 files on - # 2026-07-30) on top of the walk rm already does, just to log a number. - echo "remaining: $(ls -1d -- */ 2>/dev/null | wc -l) generation(s)" - volumeMounts: - - name: hicache - mountPath: /hicache - resources: - requests: - cpu: 10m - memory: 64Mi - # Memory only: busybox rm streams, so this just bounds the pathological - # case. No cpu limit -- the job is iowait-bound, where CFS quota cannot - # throttle, and if entries are cache-hot it would stretch the run instead. - limits: - memory: 128Mi - volumes: - - name: hicache - persistentVolumeClaim: - claimName: qwen36-27b-hicache ---- -# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/model_v1alpha1.json -apiVersion: inference.llmkube.dev/v1alpha1 -kind: Model -metadata: - name: qwen36-27b-awq -spec: - source: hf://mattbucci/Qwen3.6-27B-AWQ@f541031dfc1bf5da3f198b52b9bf44cbd5ceee49 - format: safetensors - refreshPolicy: OnChange - files: - - model.safetensors - - model-vision.safetensors - - model.safetensors.index.json - - config.json - - generation_config.json - - chat_template.jinja - - processor_config.json - - tokenizer.json - - tokenizer_config.json - hardware: - accelerator: rocm - gpu: - enabled: true - count: 1 - vendor: amd - runtime: rocm - resourceName: squat.ai/dri ---- -# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/inferenceservice_v1alpha1.json -apiVersion: inference.llmkube.dev/v1alpha1 -kind: InferenceService -metadata: - name: qwen36-27b -spec: - modelRef: qwen36-27b-awq - runtime: sglang - # Owns the R9700: more context, faster decode and working vision than vLLM at 0. - replicas: 1 - # Operator default binds :: (IPv6-only on this IPv4 cluster); override to v4. - bindAddress: 0.0.0.0 - image: ghcr.io/tanguille/sglang-rdna4:sha-cb7b76050cbf8dda9e5a78c07b24df1a783caeb3@sha256:f961fc274eaad4af065097e0921b88ae5a97a478baa0e384e6c5ee440c355b29 - containerPort: 30000 - endpoint: - # v0.9.8 maps endpoint.port to both Service port and targetPort; use the - # runtime port so generated traffic reaches SGLang directly. - port: 30000 - nodeSelector: - kubernetes.io/hostname: control-1 - podSecurityContext: - # The image's own USER 10001:10001 owns /home/sglang/.cache 0700 (flashinfer's JIT - # workspace lives there). The operator drops ALL capabilities on the container, so - # runAsUser: 0 no longer gets CAP_DAC_OVERRIDE to force its way past that ownership -- - # root became just another uid. Match the image's own user instead of overriding it. - runAsUser: 10001 - runAsGroup: 10001 - # The triton-cache and hicache PVCs were written by every prior root-based pod, so - # they're root-owned on disk. fsGroup makes kubelet recursively chown volume mounts - # to this group at mount time -- the same fix as runAsGroup, but for PVC contents - # instead of the image's own baked files. OnRootMismatch: hicache already holds 64G; - # skip the recursive chown walk on every future restart once the root matches. - fsGroup: 10001 - fsGroupChangePolicy: OnRootMismatch - supplementalGroups: - - 44 - - 226 - seccompProfile: - type: Unconfined - modelCache: - claimName: qwen36-27b-model-cache - sglangConfig: - contextLength: 180000 - # HARD CEILING: the R9700 is shared with Jellyfin transcoding, raising this fails - # media workloads. 0.95 grows the pool 183,240 -> 261,019 tokens but OOMs on a - # 13.8K-token prefill (2026-07-25). - memFractionStatic: 0.875 - # No maxRunningRequests: the mamba clamp below always wins over a requested value - # (resolve_max_num_reqs), so setting one here would only misstate the real ceiling. - chunkedPrefillSize: 4096 - # No tensorParallelSize (1 is the default) or quantization: config.json carries - # quant_method awq and dtype bfloat16, both auto-detected; awq_marlin, the one override - # that could diverge, hard-returns False off CUDA (awq.py:388). Raising TP also means - # re-adding the flags dropped as TP=1 no-ops under extraArgs. kvCacheDtype does NOT - # auto-detect -- it buys the pool. - kvCacheDtype: fp8_e4m3 - reasoningParser: qwen3 - resources: - cpu: "2" - hostMemory: 24Gi - env: - # No *_VISIBLE_DEVICES/GPU_DEVICE_ORDINAL: the device plugin hands over one render node - # (card0 + renderD128) and torch sees one device, so "0" was the identity mapping x4. - # $HOME/.cache/flashinfer// is root-owned in the image (a build-time - # `import torch, sglang, sgl_kernel` verification step triggers flashinfer's JIT - # logger init as root, before the runtime user exists) -- uid 10001 can enter it - # (0755) but not write the JIT log, crashing at import. Redirect off $HOME entirely. - - name: FLASHINFER_WORKSPACE_BASE - value: /tmp - - name: TRITON_CACHE_DIR - value: /cache/sglang/triton - # Required at TP=1: LLMKube runs launch_server directly, bypassing the image's - # entrypoint.sh/gpu-selection.sh, the only place this is normally set. Without it the - # JIT store_cache kernel crashes on first request (kvcache.cuh:204). - - name: SGLANG_RDNA4_DISABLE_STORE_CACHE - value: "1" - # Page names encode neither the weights revision nor the engine, so this suffix MUST be - # bumped with the image digest above or the AWQ revision, or stale KV pages get served. - # SGLang's own per-model cache key (config_suffix, HiCacheFile.__init__) only covers - # --served-model-name/TP, which stays "qwen-3.6" across bumps -- it doesn't catch this. - # Its evictor only manages its own configured path, so the old suffix's directory is - # orphaned on every bump; the qwen36-27b-hicache-prune CronJob above reclaims it. - - name: SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR - value: /hicache/sglang-v0.5.16_awq-f541031d_fork-cb7b760 - # Unset defaults to unbounded. 64Gi made L3 write-only: it evicted every entry before - # it was read back. 128Gi didn't fix it either -- the store still sat at ~99% full - # continuously (2026-08-04 measurement), so it was still evicting under write - # pressure just as before, only less visibly. 192Gi gives real headroom: control-1's - # /var had 268.7Gi free and SSD_Pool (backing the zvol) had 364Gi free at time of - # writing. Deliberately above the PVC's nominal 64Gi -- hostpath enforces no quota - # and openebs-hostpath cannot expand, so this env var is the real bound. - - name: SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE - value: 192Gi - # MAX_SIZE is blind to the other tenants of control-1's shared root, so hold a floor - # above kubelet's nodefs eviction threshold: refuse writes rather than fill the disk. - # 160Gi, not 100Gi: rook-ceph's mon-c store shares this 500G filesystem and MON_DISK_LOW - # fires below 30% avail (150Gi), which a 100Gi floor guarantees (2026-08-06). - - name: SGLANG_HICACHE_FILE_BACKEND_MIN_FREE_SPACE - value: 160Gi - # Not the eviction trigger despite the log's wording: eviction fires at MAX_SIZE and - # this is the low-water mark it drains to. Higher = warmer cache, more frequent passes. - - name: SGLANG_HICACHE_FILE_BACKEND_EVICTION_RATIO - value: "0.95" - # Without this a lookup miss walks the whole store with os.scandir. The startup scan it - # adds is one os.listdir, not a per-file stat. Off by default. - - name: SGLANG_HICACHE_FILE_BACKEND_ENABLE_METADATA_CACHE - value: "1" - # The 5s default expires every entry the startup scan just inserted; -1 never expires. - - name: SGLANG_HICACHE_FILE_BACKEND_METADATA_TTL - value: "-1" - extraVolumes: - - name: triton-cache - persistentVolumeClaim: - claimName: qwen36-27b-triton-cache - - name: hicache - persistentVolumeClaim: - claimName: qwen36-27b-hicache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 8Gi - extraVolumeMounts: - - name: triton-cache - mountPath: /cache - - name: hicache - mountPath: /hicache - - name: dshm - mountPath: /dev/shm - probeOverrides: - # SGLang's health endpoint is suitable for startup; keep liveness inert so - # long model loads or transient GPU work cannot trigger a restart. - startup: - httpGet: - path: /health - port: 30000 - initialDelaySeconds: 60 - periodSeconds: 15 - timeoutSeconds: 5 - failureThreshold: 120 - liveness: - exec: - command: - - "true" - # Inert probe; hourly period avoids a pointless CRI exec every 10s. - periodSeconds: 3600 - readiness: - tcpSocket: - port: 30000 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 6 - extraArgs: - # Dropped as no-ops at TP=1 on this checkpoint: --dtype bfloat16 (auto-resolves), - # --trust-remote-code (no auto_map, no .py files), --pre-warm-nccl (gated on tp/pp/ep>1, - # bootstrap.py:109), --disable-custom-all-reduce (never built at world_size 1, :438). - - --num-continuous-decode-steps - - "16" - - --watchdog-timeout - - "600" - - --attention-backend - - triton - - --chat-template - - /opt/rdna4-inference/scripts/qwen3.6_devrole_chat_template.jinja - - --tool-call-parser - - qwen3_coder - - --disable-overlap-schedule - - --cuda-graph-backend-decode=disabled - - --cuda-graph-backend-prefill=disabled - # Mamba slots and KV tokens share one byte budget (unified_memory_pool.py), so unused - # slots are KV tokens forgone: 4.46G/60 slots = 74.3MB each vs 5.60G/183,240 KV tokens. - # 14d utilization (3643 5m samples): slots hit 32 in only 2 samples (0.055%), never higher. - # 32 matches the observed peak with zero margin, freeing 16 slots (~1.2G) to the KV pool. - - --max-mamba-cache-size - - "32" - - --served-model-name - - qwen-3.6 - - --mamba-ssm-dtype - - bfloat16 - - --max-queued-requests - - "32" - - --weight-loader-drop-cache-after-load - - --enable-hierarchical-cache - - --hicache-io-backend - - direct - - --hicache-ratio - - "1.5" - - --model-loader-extra-config - - '{"num_threads":2}' - - --enable-mixed-chunk - # Without this SGLang returns prompt_tokens_details:None and hermes logs - # cache_read_tokens=0 regardless of the real hit rate. - - --enable-cache-report - # Promote only on hit_count >= 2. page_size is pinned at 1 by mamba no_buffer, so - # write_through is ~1 file per prompt token, which halved per-request decode. - - --hicache-write-policy - - write_through_selective - - --hicache-storage-backend - - file diff --git a/kubernetes/apps/ai/llmkube/models/qwen36-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen36-27b-vllm.yaml deleted file mode 100644 index 8408191eb9..0000000000 --- a/kubernetes/apps/ai/llmkube/models/qwen36-27b-vllm.yaml +++ /dev/null @@ -1,264 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: qwen36-27b-vllm-cache - namespace: ai -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath - resources: - requests: - storage: 50Gi ---- -# Persist torch, Triton, and Inductor compile caches across restarts. -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: qwen36-27b-vllm-compile-cache - namespace: ai -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath - resources: - requests: - storage: 10Gi ---- -# L3 tier for the fs secondary tier -- the vLLM analogue of SGLang's HiCache file backend. -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: qwen36-27b-vllm-kv-offload - namespace: ai -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath - resources: - # hostpath enforces no quota, so the real bound is the free-space guard below. - requests: - storage: 128Gi ---- -# vLLM's fs tier has no evictor at v0.27.1, and control-1's root fs is shared with -# rook-ceph's mon-c store. Enforces the 160Gi floor SGLang's file backend enforces itself. -apiVersion: batch/v1 -kind: CronJob -metadata: - name: qwen36-27b-vllm-kv-offload-guard - namespace: ai -spec: - # Daily: SGLang's store took weeks to reach 147G. - schedule: "0 4 * * *" - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 2 - template: - spec: - restartPolicy: OnFailure - securityContext: - # The serving pod runs runAsUser 0, so the store is root-owned on disk. - runAsUser: 0 - containers: - - name: guard - image: ghcr.io/home-operations/busybox:1.38.0@sha256:7e2c04dd50ede647bf4a7a4c8dbd629dd4971cd139b9b88fb22bfc3c7a6c13df - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: - - /bin/sh - - -c - - | - # $$ throughout: Flux postBuild envsubst substitutes bare $VAR. - set -eu - floor=167772160 - # df, not du: the store reaches hundreds of thousands of files. - avail="$$(df -kP /kvoffload | awk 'NR==2{print $$4}')" - echo "avail: $${avail}K (floor: $${floor}K)" - if [ "$$avail" -lt "$$floor" ]; then - # ponytail: wipe-all, not LRU -- no per-entry recency on disk to sort by. - echo "below floor, wiping the L3 store" - rm -rf -- /kvoffload/* - fi - volumeMounts: - - name: kv-offload - mountPath: /kvoffload - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - memory: 128Mi - volumes: - - name: kv-offload - persistentVolumeClaim: - claimName: qwen36-27b-vllm-kv-offload ---- -# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/model_v1alpha1.json -apiVersion: inference.llmkube.dev/v1alpha1 -kind: Model -metadata: - name: qwen36-27b-vllm -spec: - # Pin the model revision for reproducible staging. QuantTrio is the boot-validated - # variant; the prior cyankiwi INT4 pin was never booted. - source: hf://QuantTrio/Qwen3.6-27B-AWQ@9b507bdc9afafb87b7898700cc2a591aa6639461 - format: safetensors - quantization: AWQ - files: - - model-00001-of-00008.safetensors - - model-00002-of-00008.safetensors - - model-00003-of-00008.safetensors - - model-00004-of-00008.safetensors - - model-00005-of-00008.safetensors - - model-00006-of-00008.safetensors - - model-00007-of-00008.safetensors - - model-00008-of-00008.safetensors - - model.safetensors.index.json - - config.json - - configuration.json - - generation_config.json - - tokenizer.json - - tokenizer_config.json - - chat_template.jinja - - merges.txt - - vocab.json - - preprocessor_config.json - - video_preprocessor_config.json - refreshPolicy: OnChange - hardware: - accelerator: rocm - memoryBudget: 34Gi - gpu: - enabled: true - count: 1 - vendor: amd - runtime: rocm - resourceName: squat.ai/dri - memory: 32Gi ---- -# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/inferenceservice_v1alpha1.json -apiVersion: inference.llmkube.dev/v1alpha1 -kind: InferenceService -metadata: - name: qwen36-27b-vllm -spec: - modelRef: qwen36-27b-vllm - runtime: vllm - # Operator default binds :: (IPv6-only on this IPv4 cluster); override to v4. - bindAddress: 0.0.0.0 - modelCache: - claimName: qwen36-27b-vllm-cache - # v0.26.0 measured identical to the nightly on KV pool and throughput (2026-07-25). - # The nightly self-reports a stale 0.23.1rc1 base tag; tags are not version-ordered. - image: vllm/vllm-openai-rocm:v0.27.1@sha256:bb44b39aea26798cce43030a98bf48efd0322ca7147367db86e38b96bd80f0e7 - # Parked: 3.9 tok/s decode on real traffic, and vision needs the pool it cannot spare. - replicas: 0 - # Above 16 the 125,041-token pool oversubscribes: TG 48 -> 35, TTFT 11s -> 85s. - parallelSlots: 16 - vllmConfig: - # Fits the measured pool below; SGLang's 180K needs ~6.55 GiB. - maxModelLen: 112000 - kvCacheDtype: fp8_e4m3 - # Off, mamba leaves 'align' mode and the offloading connector asserts. - enablePrefixCaching: true - maxNumBatchedTokens: 4096 - # HARD CEILING: shared with Jellyfin transcoding, never raise. - gpuMemoryUtilization: 0.875 - env: - - name: VLLM_ROCM_USE_AITER - value: "0" - - name: HIP_VISIBLE_DEVICES - value: "0" - - name: ROCR_VISIBLE_DEVICES - value: "0" - - name: VLLM_CACHE_ROOT - value: /cache/vllm - - name: TRITON_CACHE_DIR - value: /cache/triton - - name: TORCHINDUCTOR_CACHE_DIR - value: /cache/inductor - # Pins the fs tier's block-hash seed; unset it is random per process and restarts lose it. - - name: PYTHONHASHSEED - value: "0" - extraVolumes: - - name: compile-cache - persistentVolumeClaim: - claimName: qwen36-27b-vllm-compile-cache - - name: kv-offload - persistentVolumeClaim: - claimName: qwen36-27b-vllm-kv-offload - # The CPU offload tier mmaps 8Gi here; the 64M default EFAULTs on pre-fault. - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 10Gi - extraVolumeMounts: - - name: compile-cache - mountPath: /cache - - name: kv-offload - mountPath: /kvoffload - - name: dshm - mountPath: /dev/shm - extraArgs: - - --served-model-name - - qwen-3.6 - # Zeroes every modality limit, so the vision tower loads on the meta device. - - --language-model-only - # Use bfloat16 Mamba cache dtype to reduce memory for the long context window. - - --mamba-ssm-cache-dtype - - bfloat16 - - --trust-remote-code - # Without these the gateway gets thinking blocks and tool calls as raw text. - - --reasoning-parser - - qwen3 - - --enable-auto-tool-choice - - --tool-call-parser - - qwen3_coder - # vLLM's own "to fit into requested memory" figure at 0.875 (measured 2026-08-12). - - --kv-cache-memory-bytes - - "4691494912" - # Hierarchical KV cache: CPU tier = SGLang's --hicache-ratio (9.01 GB measured), fs tier - # = its file backend. No block_size -- Qwen3.6's hybrid groups trip a divisibility assert. - - --kv-transfer-config - - >- - {"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":8589934592,"secondary_tiers":[{"type":"fs","root_dir":"/kvoffload","locality":"LOCAL"}]}} - probeOverrides: - startup: - httpGet: - path: /health - port: 8000 - initialDelaySeconds: 60 - periodSeconds: 15 - failureThreshold: 240 - # Inert by design: long model loads and transient GPU work must not restart the pod. - liveness: - exec: - command: ["true"] - periodSeconds: 3600 - readiness: - tcpSocket: - port: 8000 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 6 - nodeSelector: - amd.com/gpu: "true" - podSecurityContext: - runAsUser: 0 - runAsGroup: 0 - supplementalGroups: [44, 226] - seccompProfile: - type: Unconfined - resources: - cpu: "2" - # 24Gi base + the 8 GiB the CPU offload tier pins. - memory: 32Gi - endpoint: - port: 8000 diff --git a/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml new file mode 100644 index 0000000000..be674401ed --- /dev/null +++ b/kubernetes/apps/ai/llmkube/models/qwen38-27b-vllm.yaml @@ -0,0 +1,311 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: qwen38-27b-vllm-model-cache +spec: + accessModes: [ReadWriteOnce] + storageClassName: openebs-hostpath + resources: + requests: + storage: 50Gi +--- +# Persist torch, Triton, and Inductor compile caches across restarts. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: qwen38-27b-vllm-compile-cache +spec: + accessModes: [ReadWriteOnce] + storageClassName: openebs-hostpath + resources: + requests: + storage: 10Gi +--- +# L3 tier for the fs secondary tier -- the vLLM analogue of SGLang's HiCache +# file backend. Ported from qwen36-27b-vllm.yaml: proven pattern, same +# hardware, same GPU-memory ceiling. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: qwen38-27b-vllm-kv-offload +spec: + accessModes: [ReadWriteOnce] + storageClassName: openebs-hostpath + resources: + # hostpath enforces no quota, so the real bound is the free-space guard below. + requests: + storage: 128Gi +--- +# vLLM's fs tier has no evictor at v0.27.1, and this node's root fs is shared with +# rook-ceph's mon store, so the tier needs an external one. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: qwen38-27b-vllm-kv-offload-guard +spec: + # Hourly, not daily: a daily window let a busy day cross the floor and sit + # there for hours. Safe to run this often only because eviction is now + # incremental (see the script) rather than a full wipe. + # ponytail: periodic sweep, not a hard cap -- move to a quota-enforcing + # storage class if the floor is ever actually breached. + schedule: "0 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + securityContext: + # The serving pod runs runAsUser 0, so the store is root-owned on disk. + runAsUser: 0 + containers: + - name: guard + image: ghcr.io/home-operations/busybox:1.38.0@sha256:7e2c04dd50ede647bf4a7a4c8dbd629dd4971cd139b9b88fb22bfc3c7a6c13df + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -c + - | + # $$ throughout: Flux postBuild envsubst substitutes bare $VAR. + set -eu + # 80Gi, not the 160Gi this started as: unreachable on a 500G + # disk with ~370G held elsewhere, so the guard wiped the whole + # tier every run. The prefix-hit rate needs the store to survive. + floor=83886080 + # df, not du: the store reaches hundreds of thousands of files. + avail="$$(df -kP /kvoffload | awk 'NR==2{print $$4}')" + echo "avail: $${avail}K (floor: $${floor}K)" + if [ "$$avail" -ge "$$floor" ]; then + echo "above floor, nothing to do" + exit 0 + fi + # Oldest-first, not a wipe: blocks are write-once so mtime is + # real recency. Walk down only until back above the floor. + for age in 30 14 7 3 1 0; do + find /kvoffload -type f -mtime +$$age -delete 2>/dev/null || true + avail="$$(df -kP /kvoffload | awk 'NR==2{print $$4}')" + echo "evicted >$${age}d, avail now $${avail}K" + if [ "$$avail" -ge "$$floor" ]; then + exit 0 + fi + done + # Still short with only <24h blocks left: the shortfall is + # elsewhere on the shared fs. Fail loudly -- a guard that + # silently misses its floor reads healthy until the node fills. + echo "floor unreachable: $${avail}K < $${floor}K, only <24h blocks left" >&2 + exit 1 + volumeMounts: + - name: kv-offload + mountPath: /kvoffload + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 128Mi + volumes: + - name: kv-offload + persistentVolumeClaim: + claimName: qwen38-27b-vllm-kv-offload +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/model_v1alpha1.json +apiVersion: inference.llmkube.dev/v1alpha1 +kind: Model +metadata: + name: qwen38-27b-vllm +spec: + source: hf://cyankiwi/Qwen3.8-27B-AWQ-INT4@63768c10df38c0395e12ef49edac1bd539eaeeea + format: safetensors + quantization: compressed-tensors + refreshPolicy: OnChange + files: + - model-00001-of-00005.safetensors + - model-00002-of-00005.safetensors + - model-00003-of-00005.safetensors + - model-00004-of-00005.safetensors + - model-00005-of-00005.safetensors + - model.safetensors.index.json + - config.json + - generation_config.json + - tokenizer.json + - tokenizer_config.json + - chat_template.jinja + - merges.txt + - vocab.json + - preprocessor_config.json + - video_preprocessor_config.json + hardware: + accelerator: rocm + memoryBudget: 28Gi + gpu: + enabled: true + count: 1 + vendor: amd + runtime: rocm + resourceName: squat.ai/dri + memory: 32Gi +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/inference.llmkube.dev/inferenceservice_v1alpha1.json +# concsweep.py, warm, agg tok/s vs the SGLang qwen-3.6 baseline (conc 1/8/16): +# 14.88/53.93/100.82 vs 14.96/36.24/34.49. Bench warm only -- cold reads ~12. +apiVersion: inference.llmkube.dev/v1alpha1 +kind: InferenceService +metadata: + name: qwen38-27b-vllm +spec: + modelRef: qwen38-27b-vllm + runtime: vllm + replicas: 1 + # Matches the proven qwen36 production concurrency ceiling; operator + # translates this to --max-num-seqs for the vllm runtime. + parallelSlots: 16 + bindAddress: 0.0.0.0 + image: vllm/vllm-openai-rocm:v0.27.1@sha256:bb44b39aea26798cce43030a98bf48efd0322ca7147367db86e38b96bd80f0e7 + modelCache: + claimName: qwen38-27b-vllm-model-cache + vllmConfig: + # Hermes hard-requires >=64K context (refuses to start below it); its + # measured percentiles are p90 94K, p95 106K, p100 112K, so this covers + # p100 and routine sessions never truncate. Reachable only with the + # explicit --kv-cache-memory below. + maxModelLen: 112000 + kvCacheDtype: fp8_e4m3 + # Confirmed working on this hybrid Mamba/GDN model (~27% hit rate under + # real traffic on the sibling qwen36 config). + enablePrefixCaching: true + maxNumBatchedTokens: 4096 + # INERT while --kv-cache-memory is set below (vLLM skips profiling and logs + # that it ignores this). Kept because dropping that flag hands sizing back + # here, and 0.875 is the hard ceiling: the ~4.6 GB free at this value is + # Jellyfin's transcode reservation, not slack, and nothing enforces it + # (docs/llm-hosting/vllm-vs-sglang-2026-07.md). + gpuMemoryUtilization: 0.875 + env: + - name: VLLM_ROCM_USE_AITER + value: "0" + - name: HIP_VISIBLE_DEVICES + value: "0" + - name: ROCR_VISIBLE_DEVICES + value: "0" + - name: VLLM_CACHE_ROOT + value: /cache/vllm + - name: TRITON_CACHE_DIR + value: /cache/triton + - name: TORCHINDUCTOR_CACHE_DIR + value: /cache/inductor + # Pins the fs tier's block-hash seed; unset it is random per process and restarts lose it. + - name: PYTHONHASHSEED + value: "0" + extraVolumes: + - name: compile-cache + persistentVolumeClaim: + claimName: qwen38-27b-vllm-compile-cache + - name: kv-offload + persistentVolumeClaim: + claimName: qwen38-27b-vllm-kv-offload + # The CPU offload tier mmaps 8Gi here; the 64M default EFAULTs on pre-fault. + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 10Gi + extraVolumeMounts: + - name: compile-cache + mountPath: /cache + - name: kv-offload + mountPath: /kvoffload + - name: dshm + mountPath: /dev/shm + extraArgs: + - --served-model-name + - qwen-3.8 + - --trust-remote-code + - --reasoning-parser + - qwen3 + - --enable-auto-tool-choice + - --tool-call-parser + - qwen3_coder + # Qwen3.5-family GDN hybrids OOM during CUDA-graph capture at vLLM's + # default 512-size ceiling regardless of gpuMemoryUtilization/ + # enablePrefixCaching (vLLM #39010 on gfx1201). Cap capture sizes to the + # production concurrency ceiling (16) instead of disabling graphs + # outright, to keep decode throughput on par with 3.6. + # Also fixes a real crash, not just perf: hybrid Mamba/GDN cache only had + # 71 blocks available at this budget, and vLLM's default max-num-seqs + # (256) overshoots it — spec.parallelSlots above must stay <= that. + - --compilation-config + - '{"cudagraph_capture_sizes": [1, 2, 4, 8, 16]}' + # Keep prior-turn reasoning in context; matches production qwen36 config. + - --default-chat-template-kwargs + - '{"preserve_thinking": true}' + # MTP speculative decoding is deliberately OMITTED: measured on this + # hardware, MTP + tool-calling grammar wedges to ~0.2 tok/s (100x + # regression) under concurrent tool traffic. Re-test that wedge before + # re-adding. + # Covers the conv-state and SSM pools both (--mamba-ssm-cache-dtype + # defaults to following this), and bfloat16 is the smallest vLLM accepts. + - --mamba-cache-dtype + - bfloat16 + # Skips profiling, so this -- not gpuMemoryUtilization -- sizes KV. The + # profiler was conservative: 3.22 GiB, which forced the old 98K ceiling. + # 5 GiB is the most that still clears Jellyfin's ~4.6 GB: 5.46 GB free, + # 156,493 tokens, 1.40x at 112K. 6 GiB gives 1.68x but only 3.41 GB free, + # under the reservation. Re-measure free VRAM before raising. + - --kv-cache-memory + - "5368709120" + # Hierarchical KV cache: CPU tier (matches SGLang's --hicache-ratio sizing + # philosophy) + fs tier (SGLang's file backend analogue). Ported from + # qwen36-27b-vllm.yaml, WITHOUT that config's --language-model-only — + # Hermes' auxiliary.vision block actively uses this model's vision tower, + # so it must stay loaded even though that costs some context headroom. + - --kv-transfer-config + - >- + {"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":8589934592,"secondary_tiers":[{"type":"fs","root_dir":"/kvoffload","locality":"LOCAL"}]}} + nodeSelector: + amd.com/gpu: "true" + podSecurityContext: + runAsUser: 0 + runAsGroup: 0 + supplementalGroups: [44, 226] + seccompProfile: + type: Unconfined + resources: + cpu: "2" + # Measured steady state is 13.3Gi (8Gi of it the shm offload tier, which is + # tmpfs and so charged here). Kept at 32Gi for the model-load spike rather + # than trimmed to the steady state -- an OOMKill mid-load throws away the + # compile cache this node takes ~40min to rebuild. + memory: 32Gi + endpoint: + port: 8000 + probeOverrides: + startup: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 60 + periodSeconds: 15 + failureThreshold: 240 + # Startup owns the load/compile window, so this only ever sees a serving + # engine. Slack on purpose: a restart costs ~2min, so only a 10-minute + # outage (20 x 30s) counts as death. + liveness: + httpGet: + path: /health + port: 8000 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 20 + readiness: + tcpSocket: + port: 8000 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 6 diff --git a/kubernetes/apps/ai/omniroute/app/helmrelease.yaml b/kubernetes/apps/ai/omniroute/app/helmrelease.yaml index 5ee3afd4ea..320da5f240 100644 --- a/kubernetes/apps/ai/omniroute/app/helmrelease.yaml +++ b/kubernetes/apps/ai/omniroute/app/helmrelease.yaml @@ -31,7 +31,7 @@ spec: DATA_DIR: /app/data PORT: &port 20128 # Only litellm's /v1/* proxy calls skip auth (unauthenticated-backend - # + non-empty-apiKey convention, same as the qwen-3.6 SGLang model); + # + non-empty-apiKey convention, same as the qwen-3.8 vLLM model); # the dashboard at the internal route below still requires # INITIAL_PASSWORD login. REQUIRE_API_KEY: "false" diff --git a/kubernetes/apps/default/karakeep/app/helmrelease.yaml b/kubernetes/apps/default/karakeep/app/helmrelease.yaml index 463e0ff80d..4d517d90cc 100644 --- a/kubernetes/apps/default/karakeep/app/helmrelease.yaml +++ b/kubernetes/apps/default/karakeep/app/helmrelease.yaml @@ -55,7 +55,7 @@ spec: # its LiteLLMModel is auto-registered (litellm-operator autoRegister) # under the InferenceService's name, not a hand-written alias. INFERENCE_TEXT_MODEL: qwen35-2b - INFERENCE_IMAGE_MODEL: qwen-3.6-fast + INFERENCE_IMAGE_MODEL: qwen-3.8-fast INFERENCE_NUM_WORKERS: "2" INFERENCE_OUTPUT_SCHEMA: "plain" INFERENCE_ENABLE_AUTO_SUMMARIZATION: "true"