test: take keys out of the legacy proxy, enterprise and mcp unit tests before moving them

.circleci/tests.yml

@@ -74,6 +74,7 @@ commands:
     steps:
       - run:
           name: Install Codecov CLI (pinned v11.3.1)
+          when: always
           command: |
             curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
             curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
@@ -90,7 +91,6 @@ commands:
             uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
   setup_test_deps:
     steps:
-      - checkout
       - install_uv
       - install_rust
       - restore_cache:
@@ -165,9 +165,6 @@ commands:
 jobs:
   unit:
     parameters:
-      tests_path:
-        type: string
-        default: tests/unit
       flag:
         type: string
         default: unit
@@ -180,27 +177,39 @@ jobs:
       pull_request_url:
         type: string
         default: ""
+      reruns:
+        type: integer
+        default: 0
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
     working_directory: ~/project
     parallelism: << parameters.shards >>
     environment:
+      COVERAGE_CORE: sysmon
       LITELLM_LOCAL_MODEL_COST_MAP: "True"
     steps:
-      - setup_test_deps
+      - checkout
       - skip_unless_relevant:
           base_ref: << parameters.base_ref >>
           pull_request_url: << parameters.pull_request_url >>
+      - setup_test_deps
       - run:
-          name: "Run << parameters.tests_path >> shard"
+          name: "Run << parameters.flag >> shard"
           no_output_timeout: 20m
           command: |
             mkdir -p test-results/<< parameters.flag >>
-            mapfile -t files < <(find << parameters.tests_path >> -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)
-            if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.tests_path >> files; nothing to run"; exit 0; fi
+            selection="$(find tests/unit -name 'test_*.py' | sort)" || { echo "test selection failed for << parameters.flag >>"; exit 1; }
+            [ -n "${selection}" ] || { echo "test selection produced no files for << parameters.flag >>"; exit 1; }
+            shard="$(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)" || { echo "circleci tests split failed for << parameters.flag >>"; exit 1; }
+            [ -n "${shard}" ] || { echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; }
+            mapfile -t files < <(printf '%s\n' "${shard}")
+            rerun_args=(-p no:rerunfailures)
+            if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
+            test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
             set +e
-            uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
+            env -i "${test_env[@]}" \
+              uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
             status=$?
             set -e
             if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
@@ -224,6 +233,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
+      - checkout
       - setup_test_deps
       - run:
           name: Checkout litellm-docs
@@ -250,16 +260,17 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - setup_test_deps
+      - checkout
       - skip_unless_relevant:
           base_ref: << parameters.base_ref >>
           pull_request_url: << parameters.pull_request_url >>
+      - setup_test_deps
       - start_postgres:
           image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
       - start_redis
       - run:
           name: Run owned integration contracts
-          command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
+          command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
           no_output_timeout: 15m
       - run:
           name: Stop owned database and Redis

.github/workflows/test-unit-proxy-db.yml

@@ -106,6 +106,7 @@ jobs:
           - test-group: proxy-server-core
             test-path: >-
               tests/proxy_unit_tests/test_proxy_server.py
+              tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
               tests/proxy_unit_tests/test_aproxy_startup.py
             workers: 4
             dist: loadscope
@@ -115,7 +116,6 @@ jobs:
               tests/proxy_unit_tests/test_proxy_config_unit_test.py
               tests/proxy_unit_tests/test_proxy_routes.py
               tests/proxy_unit_tests/test_server_root_path.py
-              tests/proxy_unit_tests/test_proxy_pass_user_config.py
               tests/proxy_unit_tests/test_proxy_token_counter.py
               tests/proxy_unit_tests/test_request_size_limit_middleware.py
               tests/proxy_unit_tests/test_multipart_bypass_repro.py

tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py

@@ -0,0 +1,78 @@
+"""Live e2e: `/utils/token_counter?call_endpoint=true` counts Gemini `contents` upstream.
+
+Google's countTokens API is the only tokenizer that knows Gemini's real token
+boundaries, so the proxy must forward `contents` to it for both the AI Studio and
+Vertex deployments and hand back the provider's `promptTokensDetails`. Claude on
+Vertex is covered by `/v1/messages/count_tokens`; this is the Gemini `contents`
+route the claude_code rows never reach
+"""
+
+from __future__ import annotations
+
+import pytest
+from e2e_config import unique_marker
+from e2e_http import require_successful_call
+from proxy_client import ProxyClient
+from pydantic import BaseModel
+
+pytestmark = pytest.mark.e2e
+
+GEMINI_DEPLOYMENTS = ("gemini-2.5-flash", "gemini-2.5-flash-vertex")
+
+
+class _Part(BaseModel):
+    text: str
+
+
+class _Content(BaseModel):
+    parts: tuple[_Part, ...]
+
+
+class _TokenCountBody(BaseModel):
+    model: str
+    contents: tuple[_Content, ...]
+
+
+class _CallEndpoint(BaseModel):
+    call_endpoint: bool = True
+
+
+class _ModalityTokens(BaseModel):
+    modality: str
+    tokenCount: int
+
+
+class _CountTokensUpstream(BaseModel):
+    totalTokens: int
+    promptTokensDetails: tuple[_ModalityTokens, ...]
+
+
+class _TokenCountResponse(BaseModel):
+    total_tokens: int
+    request_model: str
+    model_used: str
+    tokenizer_type: str
+    original_response: _CountTokensUpstream
+
+
+class TestGeminiContentsTokenCounting:
+    @pytest.mark.parametrize("model", GEMINI_DEPLOYMENTS)
+    def test_contents_are_counted_by_the_provider_endpoint(
+        self, proxy: ProxyClient, scoped_key: str, model: str
+    ) -> None:
+        text = f"Hello world, how are you doing today? {unique_marker()}"
+        body = _TokenCountBody(model=model, contents=(_Content(parts=(_Part(text=text),)),))
+
+        result = proxy.transport.send(
+            "/utils/token_counter",
+            headers=proxy.transport.bearer(scoped_key),
+            json=body,
+            params=_CallEndpoint(),
+        )
+
+        require_successful_call(result)
+        counted = _TokenCountResponse.model_validate_json(result.body)
+        assert counted.request_model == model, counted
+        assert counted.original_response.totalTokens == counted.total_tokens > 0, counted
+        assert counted.original_response.promptTokensDetails, counted
+        assert all(detail.tokenCount > 0 for detail in counted.original_response.promptTokensDetails), counted

tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py

@@ -13,7 +13,6 @@
 from dotenv import load_dotenv
 
 load_dotenv()
-import os
 
 from unittest.mock import MagicMock
 
@@ -165,9 +164,9 @@ async def test_prometheus_metric_tracking():
                 "model_name": "gpt-5-mini",  # openai model name
                 "litellm_params": {  # params for litellm completion/embedding call
                     "model": "azure/gpt-4.1-mini",
-                    "api_key": os.getenv("AZURE_AI_API_KEY"),
-                    "api_version": os.getenv("AZURE_AI_API_VERSION"),
-                    "api_base": os.getenv("AZURE_AI_API_BASE"),
+                    "api_key": "sk-azure-unit-test",
+                    "api_version": "2025-01-01-preview",
+                    "api_base": "https://unit-test.openai.azure.com",
                 },
                 "model_info": {"id": "azure-model-id"},
             },
@@ -180,9 +179,6 @@ async def test_prometheus_metric_tracking():
             },
         ],
         provider_budget_config=provider_budget_config,
-        redis_host=os.getenv("REDIS_HOST"),
-        redis_port=int(os.getenv("REDIS_PORT", 6379)),
-        redis_password=os.getenv("REDIS_PASSWORD"),
     )
 
     try:

tests/integration/routing/test_user_config_routing.py

@@ -0,0 +1,103 @@
+import json
+import uuid
+from pathlib import Path
+from typing import Final
+
+from integration._support.client import Gateway
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, wire_server
+
+USER_KEY: Final = "sk-user-supplied-" + uuid.uuid4().hex
+
+
+def _completion(request: Request) -> Reply:
+    if request.target != "/v1/chat/completions":
+        return Reply(status=404, body=b"{}")
+    body: Final = json.loads(request.body)
+    return Reply(
+        body=json.dumps(
+            {
+                "id": "chatcmpl-user-config",
+                "object": "chat.completion",
+                "created": 0,
+                "model": body["model"],
+                "choices": [
+                    {"index": 0, "message": {"role": "assistant", "content": "routed"}, "finish_reason": "stop"}
+                ],
+                "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
+            }
+        ).encode()
+    )
+
+
+def _user_config(upstream_url: str) -> dict[str, object]:
+    return {
+        "model_list": [
+            {
+                "model_name": "user-config-deployment",
+                "litellm_params": {
+                    "model": "openai/gpt-4.1-mini",
+                    "api_base": upstream_url + "/v1",
+                    "api_key": USER_KEY,
+                },
+            }
+        ],
+        "num_retries": 0,
+    }
+
+
+def _opt_in_config(directory: Path, upstream_url: str) -> Path:
+    config: Final = directory / "allow_client_side_credentials_config.yaml"
+    config.write_text(
+        json.dumps(
+            {
+                "model_list": [
+                    {
+                        "model_name": "admin-deployment",
+                        "litellm_params": {
+                            "model": "openai/gpt-4.1-mini",
+                            "api_base": upstream_url + "/v1",
+                            "api_key": "sk-admin-configured",
+                        },
+                    }
+                ],
+                "general_settings": {
+                    "master_key": "os.environ/LITELLM_MASTER_KEY",
+                    "database_url": "os.environ/DATABASE_URL",
+                    "store_model_in_db": True,
+                    "allow_client_side_credentials": True,
+                },
+            }
+        )
+    )
+    return config
+
+
+def _request_body(upstream_url: str) -> dict[str, object]:
+    return {
+        "model": "user-config-deployment",
+        "messages": [{"role": "user", "content": "user config control"}],
+        "user_config": _user_config(upstream_url),
+    }
+
+
+def test_user_config_routes_to_the_user_supplied_deployment_when_opted_in(gateway: Gateway, tmp_path: Path) -> None:
+    with wire_server(_completion) as upstream:
+        config: Final = _opt_in_config(tmp_path, upstream.url)
+        with owned_proxy(gateway, tmp_path, {}, config=config) as candidate:
+            response: Final = candidate.request("POST", "/v1/chat/completions", _request_body(upstream.url))
+        assert response.status_code == 200, response.text
+        assert response.json()["choices"][0]["message"]["content"] == "routed"
+        outbound: Final = tuple(upstream.received.get_nowait() for _ in range(upstream.received.qsize()))
+        completions: Final = tuple(request for request in outbound if request.target == "/v1/chat/completions")
+        assert len(completions) == 1, outbound
+        assert completions[0].headers["authorization"] == f"Bearer {USER_KEY}"
+        assert json.loads(completions[0].body)["model"] == "gpt-4.1-mini"
+
+
+def test_user_config_is_rejected_without_the_opt_in(gateway: Gateway) -> None:
+    with wire_server(_completion) as upstream:
+        response: Final = gateway.request("POST", "/v1/chat/completions", _request_body(upstream.url))
+        assert response.status_code == 401, response.text
+        assert "user_config is not allowed in request body" in response.text
+        assert upstream.received.empty()