Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions .github/workflows/agent-pr-review.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,16 @@ jobs:
github_token: ${{ github.token }}
ai_base_url: http://litellm.ai.svc.cluster.local/v1
ai_api_key: ${{ secrets.LITELLM_API_KEY }}
ai_model: qwen-3.6-fast # thinking-off litellm twin (see litellm configmap)
# Empty omits the field (action default sends 0.1); SGLang runs sampling_defaults=model,
# so the model's generation_config applies (temp 1.0, top_k 20, top_p 0.95) — Qwen's
# shipped sampling; near-greedy decoding is the documented repetition-loop trigger.
ai_temperature: ""
# omniroute routes to a larger free-tier model than the self-hosted 27B
# (see litellm routerSettings.fallbacks); falls back to qwen-3.6-fast
# automatically if the free provider fails or times out.
ai_model: omniroute
# omniroute's proxied models don't share SGLang's sampling_defaults, so
# an empty temperature has no known-good fallback here — pin explicitly.
ai_temperature: "0.2"
ai_response_format: json_schema
# This GPU does ~10 tok/s with ~155-220s cold prefills near ~100K ctx (validated), so the
# action's 300s default per-request timeout is too tight. Raise it.
# Covers both omniroute (observed 2-74s) and, on fallback, qwen-3.6-fast's
# ~10 tok/s cold prefill on this GPU (155-220s near ~100K ctx, validated).
ai_request_timeout_sec: "600"
verdict_policy: findings_severity_gated
inline_findings: "true"
Expand Down
3 changes: 2 additions & 1 deletion kubernetes/apps/ai/litellm/instance/models.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ spec:
proxyRef: litellm
params:
# "auto" is OmniRoute's zero-config smart-routing model id — it builds a
# virtual combo from whichever free/keyed providers are connected.
# virtual combo from whichever free/keyed providers are connected and
# picks the best-scoring one live per request (quota/health/cost/taskFit).
model: openai/auto
apiBase: http://omniroute.ai.svc.cluster.local:20128/v1
# REQUIRE_API_KEY=false internally; LiteLLM's OpenAI provider still needs
Expand Down
6 changes: 6 additions & 0 deletions kubernetes/apps/ai/litellm/instance/proxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,12 @@ spec:
# already burned 900s only deepens the contention that caused it.
retry_policy:
InternalServerErrorRetries: 2
# omniroute's free providers have no SLA and can fail/time out without
# warning; fall back to the self-hosted (slower but always-available)
# qwen-3.6-fast alias so a flaky free provider doesn't take out the PR
# review action.
fallbacks:
- omniroute: ["qwen-3.6-fast"]
route:
hostnames:
- "litellm.${SECRET_DOMAIN}"
Expand Down