diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index 25b02dd5f37a..469d52c03a74 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -76,10 +76,13 @@ so fall back to "default" (or an explicit override) to avoid a cyclic dependency {{- end }} {{/* -Get redis service name +Get redis service name. +The bundled Redis subchart only serves sentinel in "replication" architecture +(it rejects standalone + sentinel outright), and in that mode the sentinel +Service is named "-redis", not "-redis-master". */}} {{- define "litellm.redis.serviceName" -}} -{{- if and (eq .Values.redis.architecture "standalone") .Values.redis.sentinel.enabled -}} +{{- if .Values.redis.sentinel.enabled -}} {{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} {{- else -}} {{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} diff --git a/helm/litellm-helm/templates/configmap-litellm.yaml b/helm/litellm-helm/templates/configmap-litellm.yaml index acbe4e3a4b50..03e4f6202063 100644 --- a/helm/litellm-helm/templates/configmap-litellm.yaml +++ b/helm/litellm-helm/templates/configmap-litellm.yaml @@ -1,9 +1,22 @@ {{- if .Values.proxyConfigMap.create }} +{{- $config := deepCopy .Values.proxy_config }} +{{- if and .Values.redis.enabled (dig "coordination" "enabled" true .Values.redis) }} +{{- $generalSettings := (get $config "general_settings") | default dict }} +{{- if not (hasKey $generalSettings "coordination_redis") }} +{{- $coordinationRedis := dict "host" "os.environ/REDIS_HOST" "port" "os.environ/REDIS_PORT" "password" "os.environ/REDIS_PASSWORD" }} +{{- if .Values.redis.sentinel.enabled }} +{{- $sentinelNode := list (include "litellm.redis.serviceName" .) (include "litellm.redis.port" . | int) }} +{{- $coordinationRedis = dict "sentinel_nodes" (list $sentinelNode) "service_name" (default "mymaster" .Values.redis.sentinel.masterSet) "password" "os.environ/REDIS_PASSWORD" }} +{{- end }} +{{- $_ := set $generalSettings "coordination_redis" $coordinationRedis }} +{{- $_ := set $config "general_settings" $generalSettings }} +{{- end }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: name: {{ include "litellm.fullname" . }}-config data: config.yaml: | -{{ .Values.proxy_config | toYaml | indent 6 }} +{{ $config | toYaml | indent 6 }} {{- end }} diff --git a/helm/litellm-helm/tests/coordination_redis_tests.yaml b/helm/litellm-helm/tests/coordination_redis_tests.yaml new file mode 100644 index 000000000000..0b58b1e6bc8a --- /dev/null +++ b/helm/litellm-helm/tests/coordination_redis_tests.yaml @@ -0,0 +1,143 @@ +suite: test coordination redis +templates: + - configmap-litellm.yaml + - deployment.yaml +tests: + - it: should not render coordination_redis when redis is disabled + template: configmap-litellm.yaml + set: + redis.enabled: false + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: coordination_redis + + - it: should not emit redis env vars when redis is disabled + template: deployment.yaml + set: + redis.enabled: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis-master + any: true + + - it: should render coordination_redis pointing at the bundled redis when enabled + template: configmap-litellm.yaml + set: + redis.enabled: true + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "coordination_redis:\n host: os.environ/REDIS_HOST\n password: os.environ/REDIS_PASSWORD\n port: os.environ/REDIS_PORT\n" + - matchRegex: + path: data["config.yaml"] + pattern: "master_key: os.environ/PROXY_MASTER_KEY" + + - it: should emit redis env vars backing the coordination_redis os.environ refs + template: deployment.yaml + set: + redis.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis-master + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PORT + value: "6379" + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: RELEASE-NAME-redis + key: redis-password + + - it: should not render coordination_redis when coordination is opted out + template: configmap-litellm.yaml + set: + redis.enabled: true + redis.coordination.enabled: false + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: coordination_redis + + - it: should keep emitting redis env vars when coordination is opted out + template: deployment.yaml + set: + redis.enabled: true + redis.coordination.enabled: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis-master + + - it: should not clobber a user supplied coordination_redis block + template: configmap-litellm.yaml + set: + redis.enabled: true + proxy_config.general_settings.coordination_redis: + url: os.environ/COORDINATION_REDIS_URL + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "coordination_redis:\n url: os.environ/COORDINATION_REDIS_URL\n" + - notMatchRegex: + path: data["config.yaml"] + pattern: "host: os.environ/REDIS_HOST" + + - it: should render sentinel_nodes and service_name in sentinel mode + template: configmap-litellm.yaml + set: + redis.enabled: true + redis.architecture: replication + redis.sentinel.enabled: true + asserts: + # The sentinel Service the redis subchart renders is "-redis", and a + # plain client cannot speak the sentinel protocol, so host/port must not appear + - matchRegex: + path: data["config.yaml"] + pattern: "coordination_redis:\n password: os.environ/REDIS_PASSWORD\n sentinel_nodes:\n - - RELEASE-NAME-redis\n - 26379\n service_name: mymaster\n" + - notMatchRegex: + path: data["config.yaml"] + pattern: "host: os.environ/REDIS_HOST" + + - it: should carry a custom sentinel masterSet into service_name + template: configmap-litellm.yaml + set: + redis.enabled: true + redis.architecture: replication + redis.sentinel.enabled: true + redis.sentinel.masterSet: litellm-master + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "service_name: litellm-master" + + - it: should point REDIS_HOST at the sentinel service in sentinel mode + template: deployment.yaml + set: + redis.enabled: true + redis.architecture: replication + redis.sentinel.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PORT + value: "26379" diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 6e30a6af444e..d3821a547e5f 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -331,12 +331,28 @@ postgresql: # secretKeys: # userPasswordKey: password -# requires cache: true in config file -# either enable this or pass a secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL -# with cache: true to use existing redis instance +# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend +# tracking, and the pod lock manager. Enabling this deploys the bundled Redis +# subchart, wires REDIS_HOST / REDIS_PORT / REDIS_PASSWORD into the proxy, and +# renders a `general_settings.coordination_redis` block into the proxy config. +# +# To point at an existing Redis instead, leave `enabled: false` and pass a +# secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL; the proxy +# falls back to those env vars for coordination. Set `cache: true` in the proxy +# config only if you also want LLM response caching, which is independent of +# coordination +# +# When `redis.sentinel.enabled` is set, the coordination block is rendered with +# `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead +# of host/port, because a plain Redis client cannot talk to the sentinel port redis: enabled: false architecture: standalone + coordination: + # Set to false to keep the bundled Redis for response caching only and leave + # `general_settings.coordination_redis` out of the rendered config. A + # `coordination_redis` block you define yourself in `proxy_config` always wins + enabled: true # Prisma migration job settings migrationJob: diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 4319907883e2..7c281aa158b2 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -213,6 +213,10 @@ harmless no-op for the Job and authoritative for the app pods. */}} - name: DISABLE_SCHEMA_UPDATE value: "true" +{{/* These feed the proxy's coordination Redis (cross-pod rate limits, spend + tracking, pod lock manager) via its REDIS_* env fallback. An explicit + `general_settings.coordination_redis` block in proxy_config takes + precedence over anything emitted here. */}} {{- if $root.Values.redis.host }} - name: REDIS_HOST value: {{ $root.Values.redis.host | quote }} @@ -226,10 +230,11 @@ harmless no-op for the Job and authoritative for the app pods. key: {{ $root.Values.redis.passwordSecret.passwordKey | default "password" }} {{- end }} {{- if $root.Values.redis.cluster }} -{{/* The proxy's Cache() reads REDIS_CLUSTER_NODES as JSON and constructs a - RedisClusterCache when it's set (litellm/caching/caching.py:169-192). - We seed with the single configured endpoint — the cluster client - discovers the remaining nodes from CLUSTER SLOTS at startup. */}} +{{/* The proxy falls back to REDIS_CLUSTER_NODES (JSON) to build a cluster-mode + coordination client when `general_settings.coordination_redis` is absent + and no plain-Redis response cache is configured. We seed with the single + configured endpoint; the cluster client discovers the remaining nodes from + CLUSTER SLOTS at startup. */}} - name: REDIS_CLUSTER_NODES value: {{ printf "[{\"host\":%q,\"port\":%v}]" $root.Values.redis.host (int $root.Values.redis.port) | quote }} {{- end }} diff --git a/helm/litellm/tests/redis_env_tests.yaml b/helm/litellm/tests/redis_env_tests.yaml new file mode 100644 index 000000000000..684d7071b354 --- /dev/null +++ b/helm/litellm/tests/redis_env_tests.yaml @@ -0,0 +1,109 @@ +suite: test redis coordination env vars +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: gateway omits redis env vars when no host is configured + template: gateway/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_CLUSTER_NODES + any: true + + - it: gateway emits host, port and password when redis is configured + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + redis.port: 6380 + redis.passwordSecret.name: redis-secret + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PORT + value: "6380" + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: password + + - it: backend emits the same redis env vars so both pods coordinate on one redis + template: backend/deployment.yaml + set: + redis.host: redis.example.com + redis.passwordSecret.name: redis-secret + redis.passwordSecret.passwordKey: redis-password + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + + - it: gateway omits REDIS_PASSWORD for an auth-less redis + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + any: true + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + + - it: gateway seeds REDIS_CLUSTER_NODES from host and port in cluster mode + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + redis.port: 6380 + redis.cluster: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_CLUSTER_NODES + value: '[{"host":"redis.example.com","port":6380}]' + + - it: gateway omits REDIS_CLUSTER_NODES when cluster mode is off + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_CLUSTER_NODES + any: true diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 6aa5dd39cd04..a8f2d39663e5 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -100,7 +100,18 @@ database: usernameKey: username passwordKey: password -# Optional Redis (caching, rate limiting). Leave host empty to disable. +# Optional Redis. Leave host empty to disable. +# +# This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend +# tracking, and the pod lock manager. The chart emits REDIS_HOST / REDIS_PORT / +# REDIS_PASSWORD, which the proxy picks up through its coordination Redis env +# fallback. Response caching is separate and off unless you enable it in +# `proxy_config.litellm_settings.cache`. +# +# For full control, define `general_settings.coordination_redis` in +# `proxy_config` (host/port/password/username/url/ssl/startup_nodes/ +# sentinel_nodes/sentinel_password/service_name, each accepting os.environ/VAR +# refs). An explicit block overrides these env vars. # # Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster, # self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from diff --git a/terraform/litellm/README.md b/terraform/litellm/README.md index 8f09cb534070..d4b407410521 100644 --- a/terraform/litellm/README.md +++ b/terraform/litellm/README.md @@ -178,6 +178,7 @@ only where the underlying cloud forces it. | Force destroy of object store | `s3_force_destroy` | `gcs_force_destroy` | | Database deletion protection | `skip_final_snapshot` | `cloudsql_deletion_protection` | | `proxy_config` (typed YAML map) | `proxy_config` | `proxy_config` | +| Coordination Redis | `REDIS_*` from ElastiCache (automatic) | `REDIS_*` from Memorystore (automatic) | | Extra plain env per component | `gateway_extra_env`, `backend_extra_env` | `gateway_extra_env`, `backend_extra_env` | | Extra secret-backed env | `gateway_extra_secrets`, `backend_extra_secrets` (ARNs) | `gateway_extra_secrets`, `backend_extra_secrets` (resource IDs) | | Uvicorn `--workers` on gateway | `gateway_num_workers` | `gateway_num_workers` | @@ -189,6 +190,19 @@ Each module stamps its own stack-identity tag (`litellm:stack` on AWS, merges `var.tags` / `var.labels` on top. Provider `default_tags` on AWS merge on top of all of these. +Coordination Redis needs no input on either cloud. Each module provisions the +managed Redis (ElastiCache on AWS, Memorystore on GCP) and exports `REDIS_HOST`, +`REDIS_PORT` and `REDIS_SSL` (plus `REDIS_SSL_CA_CERTS` on GCP) into the gateway +and backend env. The proxy falls back to those variables to build its +coordination Redis, which backs cross-pod tpm/rpm rate limits, spend tracking +and the pod lock manager. This is independent of LLM response caching, which +stays off unless you enable `litellm_settings.cache` in `proxy_config`. + +To coordinate through a Redis the module does not manage, set +`general_settings.coordination_redis` in `var.proxy_config`. An explicit block +overrides the `REDIS_*` env fallback; see the commented example in each +stack's `examples/default/terraform.tfvars.example` + OTel is opt-in on both clouds: leave `otel_endpoint` empty and nothing OTel-related is added to the container env; set it and both gateway and backend get `LITELLM_OTEL_V2=true` plus the full `OTEL_*` block, with diff --git a/terraform/litellm/aws/examples/default/terraform.tfvars.example b/terraform/litellm/aws/examples/default/terraform.tfvars.example index 4fdfb47e6789..061ca2a9b828 100644 --- a/terraform/litellm/aws/examples/default/terraform.tfvars.example +++ b/terraform/litellm/aws/examples/default/terraform.tfvars.example @@ -56,6 +56,19 @@ env = "stage" # general_settings = { # master_key = "os.environ/LITELLM_MASTER_KEY" # database_url = "os.environ/DATABASE_URL" +# +# # Optional. The module already exports REDIS_HOST / REDIS_PORT / REDIS_SSL +# # from the ElastiCache group it provisions, and the proxy falls back to +# # those for cross-pod rate limits, spend tracking and the pod lock manager. +# # Set this block only to coordinate through a Redis the module does not +# # manage; it overrides the REDIS_* env fallback. Cluster mode takes +# # `startup_nodes` and sentinel takes `sentinel_nodes` + `service_name` +# # coordination_redis = { +# # host = "os.environ/COORDINATION_REDIS_HOST" +# # port = "os.environ/COORDINATION_REDIS_PORT" +# # password = "os.environ/COORDINATION_REDIS_PASSWORD" +# # ssl = true +# # } # } # } diff --git a/terraform/litellm/gcp/examples/default/terraform.tfvars.example b/terraform/litellm/gcp/examples/default/terraform.tfvars.example index 6358ec96e6db..4416cf0ee5de 100644 --- a/terraform/litellm/gcp/examples/default/terraform.tfvars.example +++ b/terraform/litellm/gcp/examples/default/terraform.tfvars.example @@ -51,6 +51,20 @@ env = "stage" # general_settings = { # master_key = "os.environ/LITELLM_MASTER_KEY" # database_url = "os.environ/DATABASE_URL" +# +# # Optional. The module already exports REDIS_HOST / REDIS_PORT / REDIS_SSL +# # (plus REDIS_SSL_CA_CERTS) from the Memorystore instance it provisions, and +# # the proxy falls back to those for cross-pod rate limits, spend tracking +# # and the pod lock manager. Set this block only to coordinate through a +# # Redis the module does not manage; it overrides the REDIS_* env fallback. +# # Cluster mode takes `startup_nodes` and sentinel takes `sentinel_nodes` +# # plus `service_name` +# # coordination_redis = { +# # host = "os.environ/COORDINATION_REDIS_HOST" +# # port = "os.environ/COORDINATION_REDIS_PORT" +# # password = "os.environ/COORDINATION_REDIS_PASSWORD" +# # ssl = true +# # } # } # }