diff --git a/kubernetes/apps/ai/litellm/instance/kustomization.yaml b/kubernetes/apps/ai/litellm/instance/kustomization.yaml index 42277276bc..2557b86f9b 100644 --- a/kubernetes/apps/ai/litellm/instance/kustomization.yaml +++ b/kubernetes/apps/ai/litellm/instance/kustomization.yaml @@ -9,3 +9,4 @@ resources: - models.yaml - servicemonitor.yaml - grafanadashboard.yaml + - prometheusrule.yaml diff --git a/kubernetes/apps/ai/litellm/instance/prometheusrule.yaml b/kubernetes/apps/ai/litellm/instance/prometheusrule.yaml new file mode 100644 index 0000000000..85837f98ba --- /dev/null +++ b/kubernetes/apps/ai/litellm/instance/prometheusrule.yaml @@ -0,0 +1,87 @@ +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/monitoring.coreos.com/prometheusrule_v1.json +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: litellm-rules +spec: + groups: + - name: litellm.rules + # increase() over rate(): traffic is bursty and tiny (single digits + # overnight, dozens during a Renovate burst), so a per-second rate rounds + # to noise. It also rides out the counter resets from litellm's pod rolls. + # The window is the debounce, so `for` only has to survive one bad + # evaluation -- setting it comparable to the window adds the two together + # and hides the real firing delay from anyone reading the threshold. + rules: + # A caller got an error back. Everything else here is early warning; + # this is the outage itself. + - alert: LiteLLMRequestsFailing + expr: |- + sum by (requested_model, exception_status) ( + increase(litellm_proxy_failed_requests_metric_total[15m]) + ) > 2 + for: 10m + annotations: + summary: >- + litellm is returning {{ $labels.exception_status }} for + {{ $labels.requested_model }} — the fallback chain is not covering it + labels: + severity: critical + + # The only structural rule here: every other one needs a real request to + # fail first, which at this traffic volume means hours of lag. This + # fires on zero traffic and enforces the invariant proxy.yaml's chain + # depends on -- qwen-3.8-fast must point at a live deployment. absent() + # covers the target being retired out from under the chain, == 0 covers + # scaled-to-0, crashloop and unschedulable. + # Deployment name is hardcoded: nothing derives it from proxy.yaml's + # fallback target, so the two move together or this silently stops + # watching. Critical, not warning -- qwen-3.8 shares this backend, so a + # hit means hermes has no local model either. + - alert: LiteLLMFallbackBackendDown + expr: |- + kube_deployment_status_replicas_available{namespace="ai", deployment="qwen38-27b-vllm"} == 0 + or + absent(kube_deployment_status_replicas_available{namespace="ai", deployment="qwen38-27b-vllm"}) + # replicas: 1 on a single dGPU node, so every pod roll dips to 0. + for: 10m + annotations: + summary: >- + qwen38-27b-vllm has no available replica — litellm's only fallback + target is gone and hermes has no local model + labels: + severity: critical + + # Catches what the structural rule cannot: a target that is up but + # rejects the payload. > 0 because the chain is one deep (see + # proxy.yaml) -- a single failed fallback is the whole safety net. + - alert: LiteLLMFallbackTargetUnavailable + expr: |- + sum by (requested_model, fallback_model) ( + increase(litellm_deployment_failed_fallbacks_total[30m]) + ) > 0 + for: 5m + annotations: + summary: >- + Fallback {{ $labels.fallback_model }} for {{ $labels.requested_model }} + is failing — the chain is shorter than it looks + labels: + severity: warning + + # omniroute briefly dropping to fallback is normal -- its free tier + # cycles quota every few minutes and recovers itself. Sustained use + # means it is not recovering and the self-hosted models are carrying + # traffic they were never sized for. + - alert: LiteLLMPrimaryDegraded + expr: |- + sum by (requested_model, fallback_model) ( + increase(litellm_deployment_successful_fallbacks_total[30m]) + ) > 5 + for: 5m + annotations: + summary: >- + {{ $labels.requested_model }} has leaned on fallback + {{ $labels.fallback_model }} repeatedly — the primary is not recovering + labels: + severity: warning diff --git a/kubernetes/apps/ai/litellm/instance/proxy.yaml b/kubernetes/apps/ai/litellm/instance/proxy.yaml index 0e556d6f4c..16bed4a018 100644 --- a/kubernetes/apps/ai/litellm/instance/proxy.yaml +++ b/kubernetes/apps/ai/litellm/instance/proxy.yaml @@ -55,11 +55,14 @@ spec: retry_policy: InternalServerErrorRetries: 2 # omniroute's free providers have no SLA; qwen-3.8-fast catches that. - # qwen35-2b is the last resort because a single-entry chain went hard-down - # on 2026-08-17 when the 27B was scaled to 0 -- it is iGPU-pinned, so it - # never shares control-1's dGPU. Degraded, not down. + # Deliberately one deep. qwen35-2b is not a second tier: llama.cpp splits + # contextSize across parallelSlots, so it serves a 16384-token window and + # rejects review-sized payloads, turning a 503 into a 400. Widening it costs + # iGPU-node RAM not worth spending on a last resort. (The model itself still + # runs -- karakeep uses it.) A one-deep chain is only safe while its target + # is live, which LiteLLMFallbackBackendDown enforces. fallbacks: - - omniroute: ["qwen-3.8-fast", "qwen35-2b"] + - omniroute: ["qwen-3.8-fast"] route: hostnames: - "litellm.${SECRET_DOMAIN}" diff --git a/kubernetes/apps/ai/omniroute/app/helmrelease.yaml b/kubernetes/apps/ai/omniroute/app/helmrelease.yaml index 320da5f240..3560cb89bf 100644 --- a/kubernetes/apps/ai/omniroute/app/helmrelease.yaml +++ b/kubernetes/apps/ai/omniroute/app/helmrelease.yaml @@ -22,10 +22,16 @@ spec: app: image: repository: docker.io/diegosouzapw/omniroute - # Tried main@digest 2026-08-11 and reverted: main is only dependabot - # bumps ahead of this tag (provider work lives on the `next` image, - # which has no public branch to audit) and its CI run was cancelled. - tag: 3.8.49@sha256:92c768c56e2de32c51a0621ef182835018b00b288c9bb235c5c5e4514658c1a1 + # On `next` for one upstream fix, not for novelty: 3.8.49 never + # clears the last-known-good-provider pin when its target starts + # failing, so every request re-selects the same exhausted model + # instead of failing over (upstream #10034). No tagged release + # carries it yet; move to the v3.8.50 tag once it ships. + # Renovate cannot prompt that move -- `next` is not a version it + # can diff against, so it will only ever offer digest bumps that + # walk this further along the branch. Digest-pinned so the branch + # moving does not silently move us. + tag: next@sha256:2dd8dee4c5247c372af2ebbdfb98d72b4abacca77a07dd41aa6de6aa77e527cc env: TZ: ${TIMEZONE} DATA_DIR: /app/data