From 4a346ab18ebddc328eb7fa2d835ef09af3f30641 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 19 Sep 2026 13:18:30 -0700 Subject: [PATCH 1/8] docs(proxy): document the master key boot check and rotating off sk-1234 The proxy is about to refuse to start when the master key is not set, is empty, or is sk-1234, and its error message links to /docs/proxy/master_key_rotations#proxy-refuses-to-start Add that section to the rotation page with the two ways out: swap and restart when a salt key is set or there is no database, and boot once with the local development override, re-encrypt through POST /key/regenerate, then swap when sk-1234 is also the encryption key Add dangerously_allow_unsafe_proxy and LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY to the config settings reference, and say on the master_key rows that the proxy will not start without a real key Fix the existing regenerate example, whose body key had been replaced with a virtual key placeholder. The proxy only rotates the master key when key is the current master key --- docs/proxy/config_settings.md | 9 ++-- docs/proxy/master_key_rotations.md | 67 +++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/docs/proxy/config_settings.md b/docs/proxy/config_settings.md index d3125fc41..790896596 100644 --- a/docs/proxy/config_settings.md +++ b/docs/proxy/config_settings.md @@ -138,7 +138,8 @@ general_settings: disable_budget_reservation: boolean # disable pre-request budget reservation; may allow overspend under concurrency allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only) key_management_system: google_kms # either google_kms or azure_kms - master_key: string + master_key: string # falls back to LITELLM_MASTER_KEY; the proxy will not start when the master key is unset, empty, or sk-1234 + dangerously_allow_unsafe_proxy: boolean # local development only; lets the proxy start with no master key or with sk-1234 maximum_spend_logs_retention_period: 30d # The maximum time to retain spend logs before deletion. maximum_spend_logs_retention_interval: 1d # interval in which the spend log cleanup task should run in. user_mcp_management_mode: restricted # or "view_all" @@ -297,7 +298,8 @@ The **Default** column is the value LiteLLM uses when the setting is omitted fro | disable_budget_reservation | boolean | `false` | Default `false`. Set to `true` to disable pre-request cost reservation. This can allow concurrent requests to exceed a configured budget; requests are still rejected when the budget is already exhausted. LiteLLM logs a warning while this option is enabled. See [Budget reservation](./users#budget-reservation). | | allowed_routes | array of strings | `null` (all routes) | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](/docs/proxy/public_routes#define-public-admin-only-and-allowed-routes)| | key_management_system | string | `null` | Specifies the key management system. [Doc Secret Managers](../secret) | -| master_key | string | `null` (falls back to `LITELLM_MASTER_KEY`) | The master key for the proxy [Set up Virtual Keys](virtual_keys) | +| master_key | string | `null` (falls back to `LITELLM_MASTER_KEY`) | The master key for the proxy. The proxy will not start when it is not set, is empty, or is `sk-1234`. [Set up Virtual Keys](virtual_keys), [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | +| dangerously_allow_unsafe_proxy | boolean | `false` | For local development only: if true, the proxy starts even when the master key is not set, is empty, or is `sk-1234`, and logs a warning on every boot. Also settable via the `LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY` env var. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | | database_url | string | `null` (falls back to `DATABASE_URL`) | The URL for the database connection [Set up Virtual Keys](virtual_keys) | | database_connection_pool_limit | integer | `10` | The limit for database connection pool [Setting DB Connection Pool limit](./configs.md#configure-db-pool-limits--connection-timeouts) | | database_connection_timeout | integer | `60` (seconds) | The timeout for database connections in seconds [Setting DB Connection Pool limit, timeout](./configs.md#configure-db-pool-limits--connection-timeouts) | @@ -1186,6 +1188,7 @@ router_settings: | LITELLM_CLI_SSO_CLAIM_MAP | Alias for `CLI_SSO_CLAIM_MAP` — allowlisted OIDC claims for CLI SSO attribution metadata | LITELLM_CORS_ALLOW_CREDENTIALS | Set to `true` to explicitly allow credentials in CORS responses. When not set, credentials are disabled automatically if `LITELLM_CORS_ORIGINS` is `*` (wildcard) to prevent the browser security misconfiguration of reflecting any origin with credentials | LITELLM_CORS_ORIGINS | Comma-separated list of allowed CORS origins (e.g. `https://app.example.com,https://admin.example.com`). Defaults to `*` (all origins) when not set +| LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY | For local development only: set to `true` to let the proxy start when the master key is not set, is empty, or is `sk-1234`. Same as `general_settings.dangerously_allow_unsafe_proxy`. **Default is false**. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API | LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data | LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md) @@ -1242,7 +1245,7 @@ router_settings: | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used. | PYROSCOPE_GRAFANA_USER | Optional. Grafana Cloud Pyroscope user/tenant ID for basic auth. Required when PYROSCOPE_GRAFANA_API_TOKEN is set. | PYROSCOPE_GRAFANA_API_TOKEN | Optional. Grafana Cloud API/access policy token for Pyroscope basic auth. Required when PYROSCOPE_GRAFANA_USER is set. -| LITELLM_MASTER_KEY | Master key for proxy authentication +| LITELLM_MASTER_KEY | Master key for proxy authentication. The proxy will not start when it is not set, is empty, or is `sk-1234`. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | LITELLM_MAX_BUDGET_PER_SESSION_TTL | TTL in seconds for session budget counters used by the max-budget-per-session limiter. Default is 3600 (1 hour) | LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour) | LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit) diff --git a/docs/proxy/master_key_rotations.md b/docs/proxy/master_key_rotations.md index 84ffe9d78..08f55293f 100644 --- a/docs/proxy/master_key_rotations.md +++ b/docs/proxy/master_key_rotations.md @@ -28,14 +28,14 @@ When no salt key is set, the master key doubles as the at-rest encryption key, s Before you rotate, consider setting a permanent `LITELLM_SALT_KEY` so future master-key rotations become the no-migration flow above. Set the salt key to your current master key value first (so existing data still decrypts), then rotate the master key freely afterwards. ::: -Call `POST /key/regenerate` with the current master key and the new one. +Call `POST /key/regenerate` with the current master key as `key` and the new one as `new_master_key`. The proxy only rotates the master key when `key` is the current master key; with any other value it treats the call as a regular virtual key regeneration. ```bash curl -L -X POST 'http://localhost:4000/key/regenerate' \ -H "Authorization: Bearer $LITELLM_API_KEY" \ -H 'Content-Type: application/json' \ -d '{ - "key": "sk-", + "key": "sk-", "new_master_key": "sk-PIp1h0RekR" }' ``` @@ -75,3 +75,66 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ ``` If the UI loads and your stored models and credentials resolve, the rotation is complete. + +## Proxy refuses to start on sk-1234 {#proxy-refuses-to-start} + +The proxy exits at boot with a non-zero status and prints how to fix it when the master key it resolved is not set, is empty or only whitespace, or is the literal `sk-1234`. With no master key the proxy runs without authentication and accepts every request. `sk-1234` is the example key from LiteLLM's own docs and tutorials, so anyone who can reach the proxy can guess it. + +What to do next depends on whether the old key encrypted anything in your database. It did only if the key was `sk-1234`, `LITELLM_SALT_KEY` is not set, and the proxy has a database. The startup error carries a rotation warning that links to this section in exactly that case. If your error has no such warning, follow the first case. + +### A salt key is set, or there is no database + +Nothing is encrypted with the master key, so there is nothing to rotate. This is the same swap and restart as [rotating with a salt key](#if-you-use-a-salt-key-recommended-setup). Generate a new key, [set it](#where-to-set-the-new-key), and restart. + +```bash +echo "LITELLM_MASTER_KEY=sk-$(openssl rand -hex 32)" | tee -a .env +``` + +### No salt key, and a database with stored credentials + +Your model API keys, credentials, MCP server credentials, and DB-stored environment variables are encrypted with `sk-1234` itself. If you swap the key and restart, the proxy can no longer decrypt them. They have to be re-encrypted under the new key first, and that needs a running proxy. Back up your database before you start, for the reasons given at the top of this page. + +1. Leave the old key where it is and start the proxy once with the local development override, so that it boots on `sk-1234`. With Docker or Kubernetes, add the variable to the container's environment instead. + + ```bash + export LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY=true + litellm --config config.yaml + ``` + + While the override is on, the proxy still accepts `sk-1234`, so keep it off any network you do not trust and finish these steps in one sitting. + +2. In another shell, generate the new key and keep it in a variable. + + ```bash + export NEW_MASTER_KEY="sk-$(openssl rand -hex 32)" + echo "$NEW_MASTER_KEY" + ``` + +3. Call `POST /key/regenerate` with the old master key as both the bearer token and `key`, and the new key as `new_master_key`. This is the same request as in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key), and it re-encrypts the same stored values. + + ```bash + curl -L -X POST 'http://localhost:4000/key/regenerate' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d "{ + \"key\": \"sk-1234\", + \"new_master_key\": \"$NEW_MASTER_KEY\" + }" + ``` + +4. Stop the proxy, [set `LITELLM_MASTER_KEY` to the new key](#where-to-set-the-new-key), and remove the override (`LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY`, or `general_settings.dangerously_allow_unsafe_proxy` if you used the config setting). Restart every proxy instance, then verify as described in [After rotating](#after-rotating). + +Once you are on the new key, consider moving to a dedicated salt key, as the tip in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key) describes, so that later rotations are a swap and restart. Wait until this rotation is done before you do, because setting `LITELLM_SALT_KEY` to `sk-1234` would leave your stored credentials encrypted under a publicly known value. + +### Where to set the new key + +The new key has to reach the proxy as the `LITELLM_MASTER_KEY` environment variable. A `.env` file in the working directory is only read when you run the proxy from a checkout of the repo or load the file through Docker Compose `env_file`. With a pip install, plain `docker run`, or Kubernetes, pass the variable to the process or container directly. If the `.env` file already has a `LITELLM_MASTER_KEY` line, delete the old one so that only the new value remains. + +If the old key is written as a literal under `general_settings.master_key` in your `config.yaml`, change it to read from the environment, because a literal in the config takes precedence over the environment variable. + +```yaml +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY +``` + +Changing the master key signs everyone out of the Admin UI, because UI session tokens are signed with it. Log in again with the new key. From 782fc15f5c52b074c33c375c7efbd40236ad40c9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 19 Sep 2026 13:53:24 -0700 Subject: [PATCH 2/8] docs(proxy): generate the new key first and save it only after the regenerate call The proxy's error no longer prints the save-to-.env command when sk-1234 is also the encryption key. It prints a command that only generates a key and says to save it once this guide says to, so the rotation steps now follow that order Back up the database first, generate the key without saving it anywhere the proxy reads, boot once on the old key with the override, call POST /key/regenerate, then stop the proxy right away and only then set LITELLM_MASTER_KEY. The running process still holds the old key after the call and cannot decrypt the re-encrypted rows Also warn that the regenerate response echoes the new master key in plaintext, repeat that the regenerate call must not be used when LITELLM_SALT_KEY is set, and mirror the error's "make sure the config reads the key from the environment" wording --- docs/proxy/master_key_rotations.md | 45 ++++++++++++++++++------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/proxy/master_key_rotations.md b/docs/proxy/master_key_rotations.md index 08f55293f..a2e0613ab 100644 --- a/docs/proxy/master_key_rotations.md +++ b/docs/proxy/master_key_rotations.md @@ -80,49 +80,58 @@ If the UI loads and your stored models and credentials resolve, the rotation is The proxy exits at boot with a non-zero status and prints how to fix it when the master key it resolved is not set, is empty or only whitespace, or is the literal `sk-1234`. With no master key the proxy runs without authentication and accepts every request. `sk-1234` is the example key from LiteLLM's own docs and tutorials, so anyone who can reach the proxy can guess it. -What to do next depends on whether the old key encrypted anything in your database. It did only if the key was `sk-1234`, `LITELLM_SALT_KEY` is not set, and the proxy has a database. The startup error carries a rotation warning that links to this section in exactly that case. If your error has no such warning, follow the first case. +What to do next depends on whether the old key encrypted anything in your database. It did only if the key was `sk-1234`, `LITELLM_SALT_KEY` is not set, and the proxy has a database. In that case the startup error links to this section and prints a command that only generates a key, with a note to save it once this guide says to. In every other case its steps generate a key and save it to `.env`, and you follow the first case. ### A salt key is set, or there is no database -Nothing is encrypted with the master key, so there is nothing to rotate. This is the same swap and restart as [rotating with a salt key](#if-you-use-a-salt-key-recommended-setup). Generate a new key, [set it](#where-to-set-the-new-key), and restart. +Nothing is encrypted with the master key, so there is nothing to rotate, and the steps the error prints are all you need. If the key came from `general_settings.master_key`, make sure your config reads the key from the environment. Then generate a key and save it to `.env`, and start the proxy again. [Where to set the new key](#where-to-set-the-new-key) covers setups that do not read a `.env` file. ```bash echo "LITELLM_MASTER_KEY=sk-$(openssl rand -hex 32)" | tee -a .env ``` +This is the same swap and restart as [rotating with a salt key](#if-you-use-a-salt-key-recommended-setup). Do not call `POST /key/regenerate` with `new_master_key` when `LITELLM_SALT_KEY` is set. As the warning at the top of this page explains, that leaves your stored credentials unreadable under both keys. + ### No salt key, and a database with stored credentials -Your model API keys, credentials, MCP server credentials, and DB-stored environment variables are encrypted with `sk-1234` itself. If you swap the key and restart, the proxy can no longer decrypt them. They have to be re-encrypted under the new key first, and that needs a running proxy. Back up your database before you start, for the reasons given at the top of this page. +Your model API keys, credentials, MCP server credentials, and DB-stored environment variables are encrypted with `sk-1234` itself. If you swap the key and restart, the proxy can no longer decrypt them. They have to be re-encrypted under the new key first, and that needs a proxy running on the old key. + +:::warning Do not save the new key yet +Keep the new key out of `.env`, the `LITELLM_MASTER_KEY` environment variable, and `config.yaml` until the regenerate call in step 4 has succeeded. If it is in place any earlier, the next boot starts on the new key before anything is re-encrypted, so the proxy cannot decrypt your stored credentials and the regenerate call does not rotate anything. +::: + +1. Back up your database. The regenerate call deletes and recreates model rows and skips any credential row that fails, so a backup is the only clean way back from a partial failure. -1. Leave the old key where it is and start the proxy once with the local development override, so that it boots on `sk-1234`. With Docker or Kubernetes, add the variable to the container's environment instead. +2. Generate the new key with the command from the error. Copy the output somewhere safe, such as your secret manager, and nowhere the proxy reads from. ```bash - export LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY=true - litellm --config config.yaml + echo "sk-$(openssl rand -hex 32)" ``` - While the override is on, the proxy still accepts `sk-1234`, so keep it off any network you do not trust and finish these steps in one sitting. - -2. In another shell, generate the new key and keep it in a variable. +3. Leave the old key where it is and start the proxy once with the local development override, so that it boots on `sk-1234`. With Docker or Kubernetes, add the variable to the container's environment instead. ```bash - export NEW_MASTER_KEY="sk-$(openssl rand -hex 32)" - echo "$NEW_MASTER_KEY" + export LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY=true + litellm --config config.yaml ``` -3. Call `POST /key/regenerate` with the old master key as both the bearer token and `key`, and the new key as `new_master_key`. This is the same request as in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key), and it re-encrypts the same stored values. + While the override is on, the proxy still accepts `sk-1234`, so keep it off any network you do not trust and finish these steps in one sitting. + +4. In another shell, call `POST /key/regenerate` with the old master key as both the bearer token and `key`, and the key from step 2 as `new_master_key`. This is the same request as in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key), and it re-encrypts the same stored values. ```bash curl -L -X POST 'http://localhost:4000/key/regenerate' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ - -d "{ - \"key\": \"sk-1234\", - \"new_master_key\": \"$NEW_MASTER_KEY\" - }" + -d '{ + "key": "sk-1234", + "new_master_key": "sk-" + }' ``` -4. Stop the proxy, [set `LITELLM_MASTER_KEY` to the new key](#where-to-set-the-new-key), and remove the override (`LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY`, or `general_settings.dangerously_allow_unsafe_proxy` if you used the config setting). Restart every proxy instance, then verify as described in [After rotating](#after-rotating). + The response echoes the new master key in plaintext in the `key`, `token`, and `key_name` fields, so do not paste it into tickets, chat, or logs. + +5. Stop the proxy as soon as the call returns. The running process still holds the old key and can no longer decrypt the rows it just re-encrypted, so it should not keep serving traffic. Now [set `LITELLM_MASTER_KEY` to the new key](#where-to-set-the-new-key) and remove the override (`LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY`, or `general_settings.dangerously_allow_unsafe_proxy` if you used the config setting). Start every proxy instance again, then verify as described in [After rotating](#after-rotating). Once you are on the new key, consider moving to a dedicated salt key, as the tip in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key) describes, so that later rotations are a swap and restart. Wait until this rotation is done before you do, because setting `LITELLM_SALT_KEY` to `sk-1234` would leave your stored credentials encrypted under a publicly known value. @@ -130,7 +139,7 @@ Once you are on the new key, consider moving to a dedicated salt key, as the tip The new key has to reach the proxy as the `LITELLM_MASTER_KEY` environment variable. A `.env` file in the working directory is only read when you run the proxy from a checkout of the repo or load the file through Docker Compose `env_file`. With a pip install, plain `docker run`, or Kubernetes, pass the variable to the process or container directly. If the `.env` file already has a `LITELLM_MASTER_KEY` line, delete the old one so that only the new value remains. -If the old key is written as a literal under `general_settings.master_key` in your `config.yaml`, change it to read from the environment, because a literal in the config takes precedence over the environment variable. +If the old key is written as a literal under `general_settings.master_key`, make sure your `config.yaml` reads the key from the environment, because a literal in the config takes precedence over the environment variable. ```yaml general_settings: From 669077f20bf19980df08a760598c17a8a7117851 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 19 Sep 2026 14:23:57 -0700 Subject: [PATCH 3/8] docs(proxy): cover the refusal variant for an already exported master key The startup error now picks its "set a new key" step by whether LITELLM_MASTER_KEY is already set in the proxy's environment. When it is not set, the error still prints the command that generates a key and appends it to .env. When it is set to an unsafe value, the error prints a generate-only command and says to put the new key in place of the current value wherever that is set Show both commands in the no-rotation case and explain why appending to .env does not work there: the proxy loads .env without overriding, so a value already exported in the environment wins. Say the same in "Where to set the new key", and stop describing the rotation case by the generate-only command, since it is no longer unique to it --- docs/proxy/master_key_rotations.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/proxy/master_key_rotations.md b/docs/proxy/master_key_rotations.md index a2e0613ab..a5adabe3b 100644 --- a/docs/proxy/master_key_rotations.md +++ b/docs/proxy/master_key_rotations.md @@ -80,17 +80,25 @@ If the UI loads and your stored models and credentials resolve, the rotation is The proxy exits at boot with a non-zero status and prints how to fix it when the master key it resolved is not set, is empty or only whitespace, or is the literal `sk-1234`. With no master key the proxy runs without authentication and accepts every request. `sk-1234` is the example key from LiteLLM's own docs and tutorials, so anyone who can reach the proxy can guess it. -What to do next depends on whether the old key encrypted anything in your database. It did only if the key was `sk-1234`, `LITELLM_SALT_KEY` is not set, and the proxy has a database. In that case the startup error links to this section and prints a command that only generates a key, with a note to save it once this guide says to. In every other case its steps generate a key and save it to `.env`, and you follow the first case. +What to do next depends on whether the old key encrypted anything in your database. It did only if the key was `sk-1234`, `LITELLM_SALT_KEY` is not set, and the proxy has a database. In that case the startup error links to this section and tells you to rotate the key by following this guide, saving the new key only once the guide says to. In every other case the error prints its own steps for setting a new key, and you follow the first case. ### A salt key is set, or there is no database -Nothing is encrypted with the master key, so there is nothing to rotate, and the steps the error prints are all you need. If the key came from `general_settings.master_key`, make sure your config reads the key from the environment. Then generate a key and save it to `.env`, and start the proxy again. [Where to set the new key](#where-to-set-the-new-key) covers setups that do not read a `.env` file. +Nothing is encrypted with the master key, so there is nothing to rotate, and the steps the error prints are all you need. If the key came from `general_settings.master_key`, make sure your config reads the key from the environment. The step that sets the new key depends on whether `LITELLM_MASTER_KEY` is already set in the proxy's environment. + +When the variable is not set at all, the error prints a command that generates a key and saves it to `.env`. [Where to set the new key](#where-to-set-the-new-key) covers setups that do not read a `.env` file. ```bash echo "LITELLM_MASTER_KEY=sk-$(openssl rand -hex 32)" | tee -a .env ``` -This is the same swap and restart as [rotating with a salt key](#if-you-use-a-salt-key-recommended-setup). Do not call `POST /key/regenerate` with `new_master_key` when `LITELLM_SALT_KEY` is set. As the warning at the top of this page explains, that leaves your stored credentials unreadable under both keys. +When the variable is already set to an unsafe value (`sk-1234` or empty), the error prints a command that only generates a key. Put the new key in place of the current `LITELLM_MASTER_KEY` value wherever that is set: a shell export, your container or deployment environment, or its line in `.env`. Do not just add it to `.env`. The proxy loads `.env` without overriding variables that already exist, so a value already exported in the environment wins and an appended line would never take effect. + +```bash +echo "sk-$(openssl rand -hex 32)" +``` + +Then start the proxy again. This is the same swap and restart as [rotating with a salt key](#if-you-use-a-salt-key-recommended-setup). Do not call `POST /key/regenerate` with `new_master_key` when `LITELLM_SALT_KEY` is set. As the warning at the top of this page explains, that leaves your stored credentials unreadable under both keys. ### No salt key, and a database with stored credentials @@ -137,7 +145,9 @@ Once you are on the new key, consider moving to a dedicated salt key, as the tip ### Where to set the new key -The new key has to reach the proxy as the `LITELLM_MASTER_KEY` environment variable. A `.env` file in the working directory is only read when you run the proxy from a checkout of the repo or load the file through Docker Compose `env_file`. With a pip install, plain `docker run`, or Kubernetes, pass the variable to the process or container directly. If the `.env` file already has a `LITELLM_MASTER_KEY` line, delete the old one so that only the new value remains. +The new key has to reach the proxy as the `LITELLM_MASTER_KEY` environment variable. A `.env` file in the working directory is only read when you run the proxy from a checkout of the repo or load the file through Docker Compose `env_file`. With a pip install, plain `docker run`, or Kubernetes, pass the variable to the process or container directly. + +If `LITELLM_MASTER_KEY` is already set somewhere, such as a shell export, the container or deployment environment, or a line in `.env`, replace the value there instead of appending a new line to `.env`. The proxy loads `.env` without overriding variables that are already in the environment, so an exported value wins over the file. If the old key is written as a literal under `general_settings.master_key`, make sure your `config.yaml` reads the key from the environment, because a literal in the config takes precedence over the environment variable. From 712cfa491fa617dc0799debaebfcb39937cb8378 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 19 Sep 2026 17:29:41 -0700 Subject: [PATCH 4/8] docs(proxy): replace the override and regenerate steps with the boot-time migration Getting out of a refused boot no longer needs the local development override or a call to POST /key/regenerate. On refusal the proxy counts the stored values that decrypt under the unsafe key, and when there are any it tells the user to set LITELLM_MIGRATE_FROM_MASTER_KEY to the old key next to a new LITELLM_MASTER_KEY. The next boot re-encrypts those values before serving traffic and logs when the variable can be deleted Rewrite the section around that: the case where nothing is encrypted with the old key, the migration steps with the log lines to expect, what happens when the variable is left set, when the master key is still unsafe, when a value changes during the migration, and why LITELLM_SALT_KEY must not be added halfway. Add a short subsection on what the migration covers and that it also works as an offline alternative to the regenerate call, and point to it from the existing regenerate section Add LITELLM_MIGRATE_FROM_MASTER_KEY to the environment variable reference. The general POST /key/regenerate docs for a running proxy and the salt key warning stay as they were --- docs/proxy/config_settings.md | 1 + docs/proxy/master_key_rotations.md | 71 ++++++++++++++++++------------ 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/docs/proxy/config_settings.md b/docs/proxy/config_settings.md index 790896596..b887d01e5 100644 --- a/docs/proxy/config_settings.md +++ b/docs/proxy/config_settings.md @@ -1246,6 +1246,7 @@ router_settings: | PYROSCOPE_GRAFANA_USER | Optional. Grafana Cloud Pyroscope user/tenant ID for basic auth. Required when PYROSCOPE_GRAFANA_API_TOKEN is set. | PYROSCOPE_GRAFANA_API_TOKEN | Optional. Grafana Cloud API/access policy token for Pyroscope basic auth. Required when PYROSCOPE_GRAFANA_USER is set. | LITELLM_MASTER_KEY | Master key for proxy authentication. The proxy will not start when it is not set, is empty, or is `sk-1234`. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) +| LITELLM_MIGRATE_FROM_MASTER_KEY | The previous master key. When set together with a new, safe `LITELLM_MASTER_KEY` and no `LITELLM_SALT_KEY`, the proxy re-encrypts every stored value that decrypts under the previous key at boot, before serving traffic, and logs when the variable can be deleted. Leaving it set afterwards is a no-op. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | LITELLM_MAX_BUDGET_PER_SESSION_TTL | TTL in seconds for session budget counters used by the max-budget-per-session limiter. Default is 3600 (1 hour) | LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour) | LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit) diff --git a/docs/proxy/master_key_rotations.md b/docs/proxy/master_key_rotations.md index a5adabe3b..aa570427a 100644 --- a/docs/proxy/master_key_rotations.md +++ b/docs/proxy/master_key_rotations.md @@ -10,7 +10,7 @@ If you set a [salt key](./prod.md#set-the-salt-key), the proxy decrypts stored c Back up your database before either flow. Model re-encryption deletes and recreates rows rather than updating them in place, and credential re-encryption skips any row that fails, so a partial failure can strand data. A backup lets you revert cleanly. -Virtual keys are stored hashed, not encrypted, so they keep working after either rotation. Only models stored in the database (`store_model_in_db`) are re-encrypted by the regenerate flow; models defined in your config file are not affected. This is available on the open-source build; master-key rotation is not gated behind the enterprise tier. There is no Admin UI flow for this; rotation is API-only by design. +Virtual keys are stored hashed, not encrypted, so they keep working after either rotation. Only models stored in the database (`store_model_in_db`) are re-encrypted by the regenerate flow; models defined in your config file are not affected. This is available on the open-source build; master-key rotation is not gated behind the enterprise tier. There is no Admin UI flow for this by design; you rotate through the API or at boot. ## If you use a salt key (recommended setup) @@ -28,6 +28,8 @@ When no salt key is set, the master key doubles as the at-rest encryption key, s Before you rotate, consider setting a permanent `LITELLM_SALT_KEY` so future master-key rotations become the no-migration flow above. Set the salt key to your current master key value first (so existing data still decrypts), then rotate the master key freely afterwards. ::: +You can do this against a running proxy with `POST /key/regenerate`, as described below, or at boot with `LITELLM_MIGRATE_FROM_MASTER_KEY`, which needs no running proxy and [covers more stored values](#boot-time-migration). + Call `POST /key/regenerate` with the current master key as `key` and the new one as `new_master_key`. The proxy only rotates the master key when `key` is the current master key; with any other value it treats the call as a regular virtual key regeneration. ```bash @@ -80,11 +82,11 @@ If the UI loads and your stored models and credentials resolve, the rotation is The proxy exits at boot with a non-zero status and prints how to fix it when the master key it resolved is not set, is empty or only whitespace, or is the literal `sk-1234`. With no master key the proxy runs without authentication and accepts every request. `sk-1234` is the example key from LiteLLM's own docs and tutorials, so anyone who can reach the proxy can guess it. -What to do next depends on whether the old key encrypted anything in your database. It did only if the key was `sk-1234`, `LITELLM_SALT_KEY` is not set, and the proxy has a database. In that case the startup error links to this section and tells you to rotate the key by following this guide, saving the new key only once the guide says to. In every other case the error prints its own steps for setting a new key, and you follow the first case. +What to do next depends on whether the old key encrypted anything in your database. The proxy works that out for you. When a database is configured and `LITELLM_SALT_KEY` is not set, it connects during the refusal and counts the stored values that decrypt under the unsafe key, and the steps it prints depend on the result. -### A salt key is set, or there is no database +### Nothing is encrypted with the old key -Nothing is encrypted with the master key, so there is nothing to rotate, and the steps the error prints are all you need. If the key came from `general_settings.master_key`, make sure your config reads the key from the environment. The step that sets the new key depends on whether `LITELLM_MASTER_KEY` is already set in the proxy's environment. +This is the case when the count is 0, when there is no database, or when `LITELLM_SALT_KEY` is set. There is nothing to migrate, so the error only tells you to replace the key. If the key came from `general_settings.master_key`, it first tells you to make sure your config reads the key from the environment. The step that sets the new key depends on whether `LITELLM_MASTER_KEY` is already set in the proxy's environment. When the variable is not set at all, the error prints a command that generates a key and saves it to `.env`. [Where to set the new key](#where-to-set-the-new-key) covers setups that do not read a `.env` file. @@ -98,50 +100,61 @@ When the variable is already set to an unsafe value (`sk-1234` or empty), the er echo "sk-$(openssl rand -hex 32)" ``` -Then start the proxy again. This is the same swap and restart as [rotating with a salt key](#if-you-use-a-salt-key-recommended-setup). Do not call `POST /key/regenerate` with `new_master_key` when `LITELLM_SALT_KEY` is set. As the warning at the top of this page explains, that leaves your stored credentials unreadable under both keys. +Then start the proxy again. This is the same swap and restart as [rotating with a salt key](#if-you-use-a-salt-key-recommended-setup). Do not call `POST /key/regenerate` with `new_master_key` when `LITELLM_SALT_KEY` is set. As the warning at the top of this page explains, that leaves your stored credentials unreadable. -### No salt key, and a database with stored credentials +### The database holds values encrypted with the old key -Your model API keys, credentials, MCP server credentials, and DB-stored environment variables are encrypted with `sk-1234` itself. If you swap the key and restart, the proxy can no longer decrypt them. They have to be re-encrypted under the new key first, and that needs a proxy running on the old key. +When no salt key is set, the master key also encrypts the credentials stored in your database, so replacing it alone would make them unreadable. The error says so, with the number of values it found: -:::warning Do not save the new key yet -Keep the new key out of `.env`, the `LITELLM_MASTER_KEY` environment variable, and `config.yaml` until the regenerate call in step 4 has succeeded. If it is in place any earlier, the next boot starts on the new key before anything is re-encrypted, so the proxy cannot decrypt your stored credentials and the regenerate call does not rotate anything. -::: +```text +Your database holds 6 value(s) encrypted with this master key, which encrypts stored +credentials while LITELLM_SALT_KEY is not set. Replacing the key alone makes them unreadable, so also tell +the proxy which key to migrate from: +``` + +If the database cannot be reached during the refusal, the error says "Your database could not be checked for values encrypted with this master key" and prints the same steps. Following them is safe either way, because the migration does nothing when there is nothing to migrate. -1. Back up your database. The regenerate call deletes and recreates model rows and skips any credential row that fails, so a backup is the only clean way back from a partial failure. +You tell the proxy which key to migrate from with `LITELLM_MIGRATE_FROM_MASTER_KEY`, and the next boot re-encrypts the stored values under the new master key. You do not need a running proxy, the local development override, or a call to `POST /key/regenerate`. Back up your database first, as with any rotation on this page. If the old key came from `general_settings.master_key`, the error's first step is to make your config read the key from the environment, as shown in [Where to set the new key](#where-to-set-the-new-key). -2. Generate the new key with the command from the error. Copy the output somewhere safe, such as your secret manager, and nowhere the proxy reads from. +1. Set `LITELLM_MIGRATE_FROM_MASTER_KEY` to the old key in the same place as `LITELLM_MASTER_KEY`: a shell export, your container or deployment environment, or `.env`. Use an empty value, `LITELLM_MIGRATE_FROM_MASTER_KEY=`, when the old key was empty. ```bash - echo "sk-$(openssl rand -hex 32)" + export LITELLM_MIGRATE_FROM_MASTER_KEY=sk-1234 ``` -3. Leave the old key where it is and start the proxy once with the local development override, so that it boots on `sk-1234`. With Docker or Kubernetes, add the variable to the container's environment instead. +2. Generate a new key and set it as `LITELLM_MASTER_KEY`. When the variable is already set, put the new key in place of the current value. ```bash - export LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY=true - litellm --config config.yaml + echo "sk-$(openssl rand -hex 32)" ``` - While the override is on, the proxy still accepts `sk-1234`, so keep it off any network you do not trust and finish these steps in one sitting. - -4. In another shell, call `POST /key/regenerate` with the old master key as both the bearer token and `key`, and the key from step 2 as `new_master_key`. This is the same request as in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key), and it re-encrypts the same stored values. + When nothing sets `LITELLM_MASTER_KEY` yet, which happens when the old key was a literal in `config.yaml`, the error prints commands that save both values to `.env` instead. ```bash - curl -L -X POST 'http://localhost:4000/key/regenerate' \ - -H 'Authorization: Bearer sk-1234' \ - -H 'Content-Type: application/json' \ - -d '{ - "key": "sk-1234", - "new_master_key": "sk-" - }' + echo 'LITELLM_MIGRATE_FROM_MASTER_KEY=sk-1234' | tee -a .env + echo "LITELLM_MASTER_KEY=sk-$(openssl rand -hex 32)" | tee -a .env ``` - The response echoes the new master key in plaintext in the `key`, `token`, and `key_name` fields, so do not paste it into tickets, chat, or logs. +3. Start the proxy again. Right after the database connects, and before any traffic is served, the proxy re-encrypts every stored value that decrypts under the previous key. It logs this at WARNING, so the lines show up at the default log level, among the first lines of the log. + + ```text + Re-encrypting 6 stored value(s) from the LITELLM_MIGRATE_FROM_MASTER_KEY key to the new master key. + Done re-encrypting 6 stored value(s) with the new master key. You may now delete the LITELLM_MIGRATE_FROM_MASTER_KEY environment variable. + ``` + +4. Delete `LITELLM_MIGRATE_FROM_MASTER_KEY`, then verify as described in [After rotating](#after-rotating). Leaving the variable set does no harm. Each boot then logs one notice: "LITELLM_MIGRATE_FROM_MASTER_KEY is still set, but nothing in the database is left to migrate from that key. You may now delete LITELLM_MIGRATE_FROM_MASTER_KEY." + +`LITELLM_MIGRATE_FROM_MASTER_KEY` only takes effect together with a safe master key. If you set it while `LITELLM_MASTER_KEY` is still unset, empty, or `sk-1234`, the proxy refuses to start again. + +If a stored value is edited while the migration runs, that value is left as it was and the log says "Re-encrypted N stored value(s), but M are still encrypted with the previous key because they changed during the migration. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy to migrate them." Do what it says and restart once more. + +Do not add `LITELLM_SALT_KEY` during these steps. With a salt key set the proxy expects stored values to be encrypted with the salt key, so it skips the migration and logs that there is nothing to migrate, and the values still encrypted with the old key stay unreadable. Once the migration is done, consider moving to a dedicated salt key, as the tip in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key) describes, so that later rotations are a swap and restart. + +### What the boot-time migration covers {#boot-time-migration} -5. Stop the proxy as soon as the call returns. The running process still holds the old key and can no longer decrypt the rows it just re-encrypted, so it should not keep serving traffic. Now [set `LITELLM_MASTER_KEY` to the new key](#where-to-set-the-new-key) and remove the override (`LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY`, or `general_settings.dangerously_allow_unsafe_proxy` if you used the config setting). Start every proxy instance again, then verify as described in [After rotating](#after-rotating). +The migration works for any previous master key, not only the unsafe ones, so it is also an offline alternative to `POST /key/regenerate` with `new_master_key`: set the new `LITELLM_MASTER_KEY`, set `LITELLM_MIGRATE_FROM_MASTER_KEY` to the old one, and restart. It is idempotent, and it is safe when several workers or replicas boot at once, because each row is updated only if it still holds the value that was read. Virtual keys keep working because they are stored hashed, not encrypted. With no database connected, the proxy logs that nothing was migrated. -Once you are on the new key, consider moving to a dedicated salt key, as the tip in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key) describes, so that later rotations are a swap and restart. Wait until this rotation is done before you do, because setting `LITELLM_SALT_KEY` to `sk-1234` would leave your stored credentials encrypted under a publicly known value. +It re-encrypts more than the regenerate call does: models stored in the database (`litellm_params`), credentials, `LiteLLM_Config` values (environment variables, CloudZero and Vantage settings and so on), SSO settings, cache settings, config overrides, MCP server credentials, static headers and environment variables, MCP OAuth client credentials, per-user MCP credentials and environment variables, SSO identity assertions, and the encrypted callback variables in team, key, and user metadata, including the deleted-team and deleted-key tables. ### Where to set the new key From af689fac9665e8c025b752518dea397ac7a58c9f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 19 Sep 2026 18:03:34 -0700 Subject: [PATCH 5/8] docs(proxy): match the final refusal wording and cover a failed migration Quote the refusal lead as it is now wrapped, and say the unreachable-database variant only changes the first line Add the case where a database error interrupts the migration: the proxy keeps starting, logs a WARNING that the values cannot be read until the migration succeeds, and the fix is to keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart once the database is reachable Also say that the migration goes through the primary database even with a read replica configured, and that it skips tables or columns an older schema does not have, so it works before and after a schema upgrade --- docs/proxy/master_key_rotations.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/proxy/master_key_rotations.md b/docs/proxy/master_key_rotations.md index aa570427a..9fa2fed75 100644 --- a/docs/proxy/master_key_rotations.md +++ b/docs/proxy/master_key_rotations.md @@ -107,12 +107,12 @@ Then start the proxy again. This is the same swap and restart as [rotating with When no salt key is set, the master key also encrypts the credentials stored in your database, so replacing it alone would make them unreadable. The error says so, with the number of values it found: ```text -Your database holds 6 value(s) encrypted with this master key, which encrypts stored -credentials while LITELLM_SALT_KEY is not set. Replacing the key alone makes them unreadable, so also tell -the proxy which key to migrate from: +Your database holds 6 value(s) encrypted with this master key, +which encrypts stored credentials while LITELLM_SALT_KEY is not set. Replacing the key alone makes them +unreadable, so also tell the proxy which key to migrate from: ``` -If the database cannot be reached during the refusal, the error says "Your database could not be checked for values encrypted with this master key" and prints the same steps. Following them is safe either way, because the migration does nothing when there is nothing to migrate. +If the database cannot be reached during the refusal, the first line is "Your database could not be checked for values encrypted with this master key," instead, and the same steps follow. Following them is safe either way, because the migration does nothing when there is nothing to migrate. You tell the proxy which key to migrate from with `LITELLM_MIGRATE_FROM_MASTER_KEY`, and the next boot re-encrypts the stored values under the new master key. You do not need a running proxy, the local development override, or a call to `POST /key/regenerate`. Back up your database first, as with any rotation on this page. If the old key came from `general_settings.master_key`, the error's first step is to make your config read the key from the environment, as shown in [Where to set the new key](#where-to-set-the-new-key). @@ -148,11 +148,13 @@ You tell the proxy which key to migrate from with `LITELLM_MIGRATE_FROM_MASTER_K If a stored value is edited while the migration runs, that value is left as it was and the log says "Re-encrypted N stored value(s), but M are still encrypted with the previous key because they changed during the migration. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy to migrate them." Do what it says and restart once more. +A database error during the migration does not stop the boot. The proxy keeps starting and logs at WARNING "Could not migrate stored values from the LITELLM_MIGRATE_FROM_MASTER_KEY key (). Values still encrypted with the previous key cannot be read until the migration succeeds. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy once the database is reachable." Until that restart, requests that need a stored credential still encrypted with the previous key fail, so fix the database connection and restart with the variable still set. + Do not add `LITELLM_SALT_KEY` during these steps. With a salt key set the proxy expects stored values to be encrypted with the salt key, so it skips the migration and logs that there is nothing to migrate, and the values still encrypted with the old key stay unreadable. Once the migration is done, consider moving to a dedicated salt key, as the tip in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key) describes, so that later rotations are a swap and restart. ### What the boot-time migration covers {#boot-time-migration} -The migration works for any previous master key, not only the unsafe ones, so it is also an offline alternative to `POST /key/regenerate` with `new_master_key`: set the new `LITELLM_MASTER_KEY`, set `LITELLM_MIGRATE_FROM_MASTER_KEY` to the old one, and restart. It is idempotent, and it is safe when several workers or replicas boot at once, because each row is updated only if it still holds the value that was read. Virtual keys keep working because they are stored hashed, not encrypted. With no database connected, the proxy logs that nothing was migrated. +The migration works for any previous master key, not only the unsafe ones, so it is also an offline alternative to `POST /key/regenerate` with `new_master_key`: set the new `LITELLM_MASTER_KEY`, set `LITELLM_MIGRATE_FROM_MASTER_KEY` to the old one, and restart. It is idempotent, and it is safe when several workers or replicas boot at once, because each row is updated only if it still holds the value that was read. It reads and writes through the primary database even when a read replica is configured. Tables or columns that do not exist on an older schema are skipped, so it works both before and after a schema upgrade. Virtual keys keep working because they are stored hashed, not encrypted. With no database connected, the proxy logs that nothing was migrated. It re-encrypts more than the regenerate call does: models stored in the database (`litellm_params`), credentials, `LiteLLM_Config` values (environment variables, CloudZero and Vantage settings and so on), SSO settings, cache settings, config overrides, MCP server credentials, static headers and environment variables, MCP OAuth client credentials, per-user MCP credentials and environment variables, SSO identity assertions, and the encrypted callback variables in team, key, and user metadata, including the deleted-team and deleted-key tables. From e38304112877bb11690ac8a1993e152f589b28c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 19 Sep 2026 18:18:03 -0700 Subject: [PATCH 6/8] docs(proxy): mark the error placeholder as code so MDX builds --- docs/proxy/master_key_rotations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proxy/master_key_rotations.md b/docs/proxy/master_key_rotations.md index 9fa2fed75..23c363f29 100644 --- a/docs/proxy/master_key_rotations.md +++ b/docs/proxy/master_key_rotations.md @@ -148,7 +148,7 @@ You tell the proxy which key to migrate from with `LITELLM_MIGRATE_FROM_MASTER_K If a stored value is edited while the migration runs, that value is left as it was and the log says "Re-encrypted N stored value(s), but M are still encrypted with the previous key because they changed during the migration. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy to migrate them." Do what it says and restart once more. -A database error during the migration does not stop the boot. The proxy keeps starting and logs at WARNING "Could not migrate stored values from the LITELLM_MIGRATE_FROM_MASTER_KEY key (). Values still encrypted with the previous key cannot be read until the migration succeeds. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy once the database is reachable." Until that restart, requests that need a stored credential still encrypted with the previous key fail, so fix the database connection and restart with the variable still set. +A database error during the migration does not stop the boot. The proxy keeps starting and logs at WARNING "Could not migrate stored values from the LITELLM_MIGRATE_FROM_MASTER_KEY key (``). Values still encrypted with the previous key cannot be read until the migration succeeds. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy once the database is reachable." Until that restart, requests that need a stored credential still encrypted with the previous key fail, so fix the database connection and restart with the variable still set. Do not add `LITELLM_SALT_KEY` during these steps. With a salt key set the proxy expects stored values to be encrypted with the salt key, so it skips the migration and logs that there is nothing to migrate, and the values still encrypted with the old key stay unreadable. Once the migration is done, consider moving to a dedicated salt key, as the tip in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key) describes, so that later rotations are a swap and restart. From fb06845373ad4773138a8d3fcc023509b0ea2f35 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 19 Sep 2026 18:21:48 -0700 Subject: [PATCH 7/8] docs(proxy): a failed master key migration stops the boot The proxy no longer keeps starting when the boot-time migration hits a database error. It logs the same WARNING and exits with a non-zero status, so a worker never serves with stored values it cannot read. The exception is a connection outage with general_settings.allow_requests_on_db_unavailable on, where the boot continues as it already does for the database connection Say that restarting with the same two variables resumes a partial migration, and move the quoted log line into a code block so its placeholder cannot be parsed as JSX --- docs/proxy/master_key_rotations.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/proxy/master_key_rotations.md b/docs/proxy/master_key_rotations.md index 23c363f29..e1dcd79e8 100644 --- a/docs/proxy/master_key_rotations.md +++ b/docs/proxy/master_key_rotations.md @@ -148,7 +148,13 @@ You tell the proxy which key to migrate from with `LITELLM_MIGRATE_FROM_MASTER_K If a stored value is edited while the migration runs, that value is left as it was and the log says "Re-encrypted N stored value(s), but M are still encrypted with the previous key because they changed during the migration. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy to migrate them." Do what it says and restart once more. -A database error during the migration does not stop the boot. The proxy keeps starting and logs at WARNING "Could not migrate stored values from the LITELLM_MIGRATE_FROM_MASTER_KEY key (``). Values still encrypted with the previous key cannot be read until the migration succeeds. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy once the database is reachable." Until that restart, requests that need a stored credential still encrypted with the previous key fail, so fix the database connection and restart with the variable still set. +A database error during the migration stops the boot, so a worker never serves traffic with stored values it cannot read. The proxy logs this at WARNING and then exits with a non-zero status: + +```text +Could not migrate stored values from the LITELLM_MIGRATE_FROM_MASTER_KEY key (). Values still encrypted with the previous key cannot be read until the migration succeeds. Keep LITELLM_MIGRATE_FROM_MASTER_KEY set and restart the proxy once the database is reachable. +``` + +Fix the database problem and start the proxy again with the same two variables. The next boot picks up where the failed one stopped: if it had migrated 2 of 6 values, the next boot logs "Re-encrypting 4 stored value(s)" followed by the "Done" line. The one exception follows the rule the proxy already applies to its database connection at boot. When `general_settings.allow_requests_on_db_unavailable` is true and the error is a connection outage, the boot continues, and the values still encrypted with the previous key stay unreadable until a later boot completes the migration. Do not add `LITELLM_SALT_KEY` during these steps. With a salt key set the proxy expects stored values to be encrypted with the salt key, so it skips the migration and logs that there is nothing to migrate, and the values still encrypted with the old key stay unreadable. Once the migration is done, consider moving to a dedicated salt key, as the tip in [If the master key is your encryption key](#if-the-master-key-is-your-encryption-key) describes, so that later rotations are a swap and restart. From 833da8f8b9f04535dad40a9d468872eaf557c28b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 19 Sep 2026 18:54:28 -0700 Subject: [PATCH 8/8] docs(proxy): rename the local development override to dangerously_permit_weak_or_unset_master_key BerriAI/litellm#42019 renamed the override so the name says exactly what it permits. The environment variable is now LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY and the YAML key is general_settings.dangerously_permit_weak_or_unset_master_key Update the general_settings example, the general_settings reference row and the environment variable row. The behaviour is unchanged --- docs/proxy/config_settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proxy/config_settings.md b/docs/proxy/config_settings.md index b887d01e5..8ff583afd 100644 --- a/docs/proxy/config_settings.md +++ b/docs/proxy/config_settings.md @@ -139,7 +139,7 @@ general_settings: allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only) key_management_system: google_kms # either google_kms or azure_kms master_key: string # falls back to LITELLM_MASTER_KEY; the proxy will not start when the master key is unset, empty, or sk-1234 - dangerously_allow_unsafe_proxy: boolean # local development only; lets the proxy start with no master key or with sk-1234 + dangerously_permit_weak_or_unset_master_key: boolean # local development only; lets the proxy start with no master key or with sk-1234 maximum_spend_logs_retention_period: 30d # The maximum time to retain spend logs before deletion. maximum_spend_logs_retention_interval: 1d # interval in which the spend log cleanup task should run in. user_mcp_management_mode: restricted # or "view_all" @@ -299,7 +299,7 @@ The **Default** column is the value LiteLLM uses when the setting is omitted fro | allowed_routes | array of strings | `null` (all routes) | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](/docs/proxy/public_routes#define-public-admin-only-and-allowed-routes)| | key_management_system | string | `null` | Specifies the key management system. [Doc Secret Managers](../secret) | | master_key | string | `null` (falls back to `LITELLM_MASTER_KEY`) | The master key for the proxy. The proxy will not start when it is not set, is empty, or is `sk-1234`. [Set up Virtual Keys](virtual_keys), [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | -| dangerously_allow_unsafe_proxy | boolean | `false` | For local development only: if true, the proxy starts even when the master key is not set, is empty, or is `sk-1234`, and logs a warning on every boot. Also settable via the `LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY` env var. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | +| dangerously_permit_weak_or_unset_master_key | boolean | `false` | For local development only: if true, the proxy starts even when the master key is not set, is empty, or is `sk-1234`, and logs a warning on every boot. Also settable via the `LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY` env var. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | | database_url | string | `null` (falls back to `DATABASE_URL`) | The URL for the database connection [Set up Virtual Keys](virtual_keys) | | database_connection_pool_limit | integer | `10` | The limit for database connection pool [Setting DB Connection Pool limit](./configs.md#configure-db-pool-limits--connection-timeouts) | | database_connection_timeout | integer | `60` (seconds) | The timeout for database connections in seconds [Setting DB Connection Pool limit, timeout](./configs.md#configure-db-pool-limits--connection-timeouts) | @@ -1188,7 +1188,7 @@ router_settings: | LITELLM_CLI_SSO_CLAIM_MAP | Alias for `CLI_SSO_CLAIM_MAP` — allowlisted OIDC claims for CLI SSO attribution metadata | LITELLM_CORS_ALLOW_CREDENTIALS | Set to `true` to explicitly allow credentials in CORS responses. When not set, credentials are disabled automatically if `LITELLM_CORS_ORIGINS` is `*` (wildcard) to prevent the browser security misconfiguration of reflecting any origin with credentials | LITELLM_CORS_ORIGINS | Comma-separated list of allowed CORS origins (e.g. `https://app.example.com,https://admin.example.com`). Defaults to `*` (all origins) when not set -| LITELLM_DANGEROUSLY_ALLOW_UNSAFE_PROXY | For local development only: set to `true` to let the proxy start when the master key is not set, is empty, or is `sk-1234`. Same as `general_settings.dangerously_allow_unsafe_proxy`. **Default is false**. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) +| LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY | For local development only: set to `true` to let the proxy start when the master key is not set, is empty, or is `sk-1234`. Same as `general_settings.dangerously_permit_weak_or_unset_master_key`. **Default is false**. [Proxy refuses to start on sk-1234](./master_key_rotations.md#proxy-refuses-to-start) | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API | LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data | LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md)