diff --git a/kubernetes/apps/ai/litellm/app/configmap.yaml b/kubernetes/apps/ai/litellm/app/configmap.yaml index 91cad929..a8fcbce3 100644 --- a/kubernetes/apps/ai/litellm/app/configmap.yaml +++ b/kubernetes/apps/ai/litellm/app/configmap.yaml @@ -15,6 +15,7 @@ data: maximum_spend_logs_retention_period: "30d" # else the spend-logs table grows forever allow_requests_on_db_unavailable: true # CNPG restart != LLM outage (keys served from cache) litellm_settings: + callbacks: ["prometheus"] # expose /metrics on the proxy port for the ServiceMonitor drop_params: true # strip OpenAI params a backend doesn't support instead of 400ing request_timeout: 600 # match the proxy/ollama timeout budget end-to-end num_retries: 2 # retry transient backend failures before surfacing to the client diff --git a/kubernetes/apps/ai/litellm/app/deployment.yaml b/kubernetes/apps/ai/litellm/app/deployment.yaml index 7842c613..690232fb 100644 --- a/kubernetes/apps/ai/litellm/app/deployment.yaml +++ b/kubernetes/apps/ai/litellm/app/deployment.yaml @@ -25,7 +25,7 @@ spec: automountServiceAccountToken: false containers: - name: litellm - image: ghcr.io/berriai/litellm-database:v1.91.2 + image: ghcr.io/berriai/litellm-database:v1.92.0 args: - "--config" - "/app/config.yaml" diff --git a/kubernetes/apps/ai/litellm/app/grafanadashboard.yaml b/kubernetes/apps/ai/litellm/app/grafanadashboard.yaml new file mode 100644 index 00000000..374ded5e --- /dev/null +++ b/kubernetes/apps/ai/litellm/app/grafanadashboard.yaml @@ -0,0 +1,19 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/grafana.integreatly.org/grafanadashboard_v1beta1.json +# LiteLLM ships no chart mixin (raw image deploy) — grafana-operator fetches the +# maintained dashboard JSON from the pinned tag; Renovate bumps the tag in the URL +# in lockstep with the litellm-database image. +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: litellm +spec: + allowCrossNamespaceImport: true + instanceSelector: + matchLabels: + grafana.internal/instance: grafana + datasources: + - inputName: DS_PROMETHEUS + datasourceName: prometheus + # renovate: datasource=docker depName=ghcr.io/berriai/litellm-database + url: https://raw.githubusercontent.com/BerriAI/litellm/v1.92.0/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json diff --git a/kubernetes/apps/ai/litellm/app/kustomization.yaml b/kubernetes/apps/ai/litellm/app/kustomization.yaml index 239bee22..bbb9bdf3 100644 --- a/kubernetes/apps/ai/litellm/app/kustomization.yaml +++ b/kubernetes/apps/ai/litellm/app/kustomization.yaml @@ -8,3 +8,5 @@ resources: - ./deployment.yaml - ./service.yaml - ./httproute.yaml + - ./servicemonitor.yaml + - ./grafanadashboard.yaml diff --git a/kubernetes/apps/ai/litellm/app/service.yaml b/kubernetes/apps/ai/litellm/app/service.yaml index 114f68c9..d283ef5f 100644 --- a/kubernetes/apps/ai/litellm/app/service.yaml +++ b/kubernetes/apps/ai/litellm/app/service.yaml @@ -3,6 +3,8 @@ apiVersion: v1 kind: Service metadata: name: litellm + labels: + app: litellm # ServiceMonitor selects on this spec: selector: app: litellm diff --git a/kubernetes/apps/ai/litellm/app/servicemonitor.yaml b/kubernetes/apps/ai/litellm/app/servicemonitor.yaml new file mode 100644 index 00000000..749f51b6 --- /dev/null +++ b/kubernetes/apps/ai/litellm/app/servicemonitor.yaml @@ -0,0 +1,22 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/servicemonitor_v1.json +# /metrics requires LiteLLM API-key auth (default since v1.85.0) — scrape with the +# master key. prometheus-operator reads the credentials Secret from this +# ServiceMonitor's own namespace (ai), where litellm-auth already lives. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: litellm +spec: + jobLabel: app + selector: + matchLabels: + app: litellm + endpoints: + - port: http + path: /metrics + authorization: + type: Bearer + credentials: + name: litellm-auth + key: LITELLM_MASTER_KEY