fix(ai): restore omniroute failover and alert on gateway failures

kubernetes/apps/ai/litellm/instance/kustomization.yaml

@@ -9,3 +9,4 @@ resources:
   - models.yaml
   - servicemonitor.yaml
   - grafanadashboard.yaml
+  - prometheusrule.yaml

kubernetes/apps/ai/litellm/instance/prometheusrule.yaml

@@ -0,0 +1,87 @@
+---
+# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/monitoring.coreos.com/prometheusrule_v1.json
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: litellm-rules
+spec:
+  groups:
+    - name: litellm.rules
+      # increase() over rate(): traffic is bursty and tiny (single digits
+      # overnight, dozens during a Renovate burst), so a per-second rate rounds
+      # to noise. It also rides out the counter resets from litellm's pod rolls.
+      # The window is the debounce, so `for` only has to survive one bad
+      # evaluation -- setting it comparable to the window adds the two together
+      # and hides the real firing delay from anyone reading the threshold.
+      rules:
+        # A caller got an error back. Everything else here is early warning;
+        # this is the outage itself.
+        - alert: LiteLLMRequestsFailing
+          expr: |-
+            sum by (requested_model, exception_status) (
+              increase(litellm_proxy_failed_requests_metric_total[15m])
+            ) > 2
+          for: 10m
+          annotations:
+            summary: >-
+              litellm is returning {{ $labels.exception_status }} for
+              {{ $labels.requested_model }} — the fallback chain is not covering it
+          labels:
+            severity: critical
+
+        # The only structural rule here: every other one needs a real request to
+        # fail first, which at this traffic volume means hours of lag. This
+        # fires on zero traffic and enforces the invariant proxy.yaml's chain
+        # depends on -- qwen-3.8-fast must point at a live deployment. absent()
+        # covers the target being retired out from under the chain, == 0 covers
+        # scaled-to-0, crashloop and unschedulable.
+        # Deployment name is hardcoded: nothing derives it from proxy.yaml's
+        # fallback target, so the two move together or this silently stops
+        # watching. Critical, not warning -- qwen-3.8 shares this backend, so a
+        # hit means hermes has no local model either.
+        - alert: LiteLLMFallbackBackendDown
+          expr: |-
+            kube_deployment_status_replicas_available{namespace="ai", deployment="qwen38-27b-vllm"} == 0
+            or
+            absent(kube_deployment_status_replicas_available{namespace="ai", deployment="qwen38-27b-vllm"})
+          # replicas: 1 on a single dGPU node, so every pod roll dips to 0.
+          for: 10m
+          annotations:
+            summary: >-
+              qwen38-27b-vllm has no available replica — litellm's only fallback
+              target is gone and hermes has no local model
+          labels:
+            severity: critical
+
+        # Catches what the structural rule cannot: a target that is up but
+        # rejects the payload. > 0 because the chain is one deep (see
+        # proxy.yaml) -- a single failed fallback is the whole safety net.
+        - alert: LiteLLMFallbackTargetUnavailable
+          expr: |-
+            sum by (requested_model, fallback_model) (
+              increase(litellm_deployment_failed_fallbacks_total[30m])
+            ) > 0
+          for: 5m
+          annotations:
+            summary: >-
+              Fallback {{ $labels.fallback_model }} for {{ $labels.requested_model }}
+              is failing — the chain is shorter than it looks
+          labels:
+            severity: warning
+
+        # omniroute briefly dropping to fallback is normal -- its free tier
+        # cycles quota every few minutes and recovers itself. Sustained use
+        # means it is not recovering and the self-hosted models are carrying
+        # traffic they were never sized for.
+        - alert: LiteLLMPrimaryDegraded
+          expr: |-
+            sum by (requested_model, fallback_model) (
+              increase(litellm_deployment_successful_fallbacks_total[30m])
+            ) > 5
+          for: 5m
+          annotations:
+            summary: >-
+              {{ $labels.requested_model }} has leaned on fallback
+              {{ $labels.fallback_model }} repeatedly — the primary is not recovering
+          labels:
+            severity: warning

kubernetes/apps/ai/litellm/instance/proxy.yaml

@@ -55,11 +55,14 @@ spec:
     retry_policy:
       InternalServerErrorRetries: 2
     # omniroute's free providers have no SLA; qwen-3.8-fast catches that.
-    # qwen35-2b is the last resort because a single-entry chain went hard-down
-    # on 2026-08-17 when the 27B was scaled to 0 -- it is iGPU-pinned, so it
-    # never shares control-1's dGPU. Degraded, not down.
+    # Deliberately one deep. qwen35-2b is not a second tier: llama.cpp splits
+    # contextSize across parallelSlots, so it serves a 16384-token window and
+    # rejects review-sized payloads, turning a 503 into a 400. Widening it costs
+    # iGPU-node RAM not worth spending on a last resort. (The model itself still
+    # runs -- karakeep uses it.) A one-deep chain is only safe while its target
+    # is live, which LiteLLMFallbackBackendDown enforces.
     fallbacks:
-      - omniroute: ["qwen-3.8-fast", "qwen35-2b"]
+      - omniroute: ["qwen-3.8-fast"]
   route:
     hostnames:
       - "litellm.${SECRET_DOMAIN}"

kubernetes/apps/ai/omniroute/app/helmrelease.yaml

@@ -22,10 +22,16 @@ spec:
           app:
             image:
               repository: docker.io/diegosouzapw/omniroute
-              # Tried main@digest 2026-08-11 and reverted: main is only dependabot
-              # bumps ahead of this tag (provider work lives on the `next` image,
-              # which has no public branch to audit) and its CI run was cancelled.
-              tag: 3.8.49@sha256:92c768c56e2de32c51a0621ef182835018b00b288c9bb235c5c5e4514658c1a1
+              # On `next` for one upstream fix, not for novelty: 3.8.49 never
+              # clears the last-known-good-provider pin when its target starts
+              # failing, so every request re-selects the same exhausted model
+              # instead of failing over (upstream #10034). No tagged release
+              # carries it yet; move to the v3.8.50 tag once it ships.
+              # Renovate cannot prompt that move -- `next` is not a version it
+              # can diff against, so it will only ever offer digest bumps that
+              # walk this further along the branch. Digest-pinned so the branch
+              # moving does not silently move us.
+              tag: next@sha256:2dd8dee4c5247c372af2ebbdfb98d72b4abacca77a07dd41aa6de6aa77e527cc
             env:
               TZ: ${TIMEZONE}
               DATA_DIR: /app/data