diff --git a/.github/workflows/agent-pr-review.yaml b/.github/workflows/agent-pr-review.yaml index 8e0d2fc833..50e78577c4 100644 --- a/.github/workflows/agent-pr-review.yaml +++ b/.github/workflows/agent-pr-review.yaml @@ -38,14 +38,16 @@ jobs: github_token: ${{ github.token }} ai_base_url: http://litellm.ai.svc.cluster.local/v1 ai_api_key: ${{ secrets.LITELLM_API_KEY }} - ai_model: qwen-3.6-fast # thinking-off litellm twin (see litellm configmap) - # Empty omits the field (action default sends 0.1); SGLang runs sampling_defaults=model, - # so the model's generation_config applies (temp 1.0, top_k 20, top_p 0.95) — Qwen's - # shipped sampling; near-greedy decoding is the documented repetition-loop trigger. - ai_temperature: "" + # omniroute routes to a larger free-tier model than the self-hosted 27B + # (see litellm routerSettings.fallbacks); falls back to qwen-3.6-fast + # automatically if the free provider fails or times out. + ai_model: omniroute + # omniroute's proxied models don't share SGLang's sampling_defaults, so + # an empty temperature has no known-good fallback here — pin explicitly. + ai_temperature: "0.2" ai_response_format: json_schema - # This GPU does ~10 tok/s with ~155-220s cold prefills near ~100K ctx (validated), so the - # action's 300s default per-request timeout is too tight. Raise it. + # Covers both omniroute (observed 2-74s) and, on fallback, qwen-3.6-fast's + # ~10 tok/s cold prefill on this GPU (155-220s near ~100K ctx, validated). ai_request_timeout_sec: "600" verdict_policy: findings_severity_gated inline_findings: "true" diff --git a/kubernetes/apps/ai/litellm/instance/models.yaml b/kubernetes/apps/ai/litellm/instance/models.yaml index 3f75a65a3e..9b0102316b 100644 --- a/kubernetes/apps/ai/litellm/instance/models.yaml +++ b/kubernetes/apps/ai/litellm/instance/models.yaml @@ -64,7 +64,8 @@ spec: proxyRef: litellm params: # "auto" is OmniRoute's zero-config smart-routing model id — it builds a - # virtual combo from whichever free/keyed providers are connected. + # virtual combo from whichever free/keyed providers are connected and + # picks the best-scoring one live per request (quota/health/cost/taskFit). model: openai/auto apiBase: http://omniroute.ai.svc.cluster.local:20128/v1 # REQUIRE_API_KEY=false internally; LiteLLM's OpenAI provider still needs diff --git a/kubernetes/apps/ai/litellm/instance/proxy.yaml b/kubernetes/apps/ai/litellm/instance/proxy.yaml index c3600e896d..47f11a3e7d 100644 --- a/kubernetes/apps/ai/litellm/instance/proxy.yaml +++ b/kubernetes/apps/ai/litellm/instance/proxy.yaml @@ -54,6 +54,12 @@ spec: # already burned 900s only deepens the contention that caused it. retry_policy: InternalServerErrorRetries: 2 + # omniroute's free providers have no SLA and can fail/time out without + # warning; fall back to the self-hosted (slower but always-available) + # qwen-3.6-fast alias so a flaky free provider doesn't take out the PR + # review action. + fallbacks: + - omniroute: ["qwen-3.6-fast"] route: hostnames: - "litellm.${SECRET_DOMAIN}"