diff --git a/.circleci/config.yml b/.circleci/config.yml index 307247651e7e..2b0a6924cce0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1330,6 +1330,57 @@ jobs: paths: - audio_coverage.xml - audio_coverage + redis_caching_unit_tests: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-xdist==3.6.1" + pip install "pytest-rerunfailures==14.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv \ + tests/local_testing/test_dual_cache.py \ + tests/local_testing/test_redis_batch_optimizations.py \ + tests/local_testing/test_router_utils.py \ + --cov=litellm --cov-report=xml \ + -x -s -v --junitxml=test-results/junit.xml \ + --durations=5 -n 2 \ + --reruns 2 --reruns-delay 1 + no_output_timeout: 20m + - run: + name: Rename the coverage files + command: | + mv coverage.xml redis_caching_coverage.xml + mv .coverage redis_caching_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - redis_caching_coverage.xml + - redis_caching_coverage installing_litellm_on_python: docker: - image: cimg/python:3.11 @@ -2868,114 +2919,6 @@ jobs: - store_test_results: path: test-results - proxy_e2e_azure_batches_tests: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - run: - name: Install Docker CLI - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - - run: - name: Install Python 3.12 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.12 -y - conda activate myenv - python --version - - run: - name: Install Poetry - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - pip install poetry - - run: - name: Install dockerize - command: | - wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start PostgreSQL Database - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=llmproxy \ - -e POSTGRES_PASSWORD=dbpassword9090 \ - -e POSTGRES_DB=litellm \ - -p 5432:5432 \ - postgres:15 - - run: - name: Wait for PostgreSQL to be ready - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: - name: Install system dependencies - command: | - sudo apt-get update -y - sudo apt-get install -y libpq-dev - - run: - name: Install Dependencies - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity - - run: - name: Setup litellm-enterprise - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - run: - name: Generate Prisma client - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - poetry run prisma generate --schema litellm/proxy/schema.prisma - - run: - name: Run Prisma migrations - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - cd litellm/proxy - poetry run prisma migrate deploy --schema schema.prisma - cd ../.. - - run: - name: Run Azure Batch E2E Tests - command: | - export PATH="$HOME/miniconda/bin:$PATH" - source $HOME/miniconda/etc/profile.d/conda.sh - conda activate myenv - export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - export USE_LOCAL_LITELLM=true - export USE_MOCK_MODELS=true - export USE_STATE_TRACKER=true - export LITELLM_LOG=DEBUG - poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \ - -vv -s -k "test_e2e_managed_batch" \ - --tb=short \ - --maxfail=3 \ - --durations=10 \ - --junitxml=test-results/junit.xml - no_output_timeout: 15m - upload-coverage: docker: - image: cimg/python:3.9 @@ -2997,7 +2940,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage + coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -3182,6 +3125,117 @@ jobs: CI=true npm run test -- --run \ --pool forks --poolOptions.forks.maxForks=8 + e2e_ui_testing: + docker: + - image: cimg/python:3.12-browsers + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + - image: cimg/postgres:16.0 + environment: + POSTGRES_USER: e2euser + POSTGRES_PASSWORD: e2epassword + POSTGRES_DB: litellm_e2e + resource_class: large + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" + CI: "true" + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }} + - run: + name: Install Python dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "prisma==0.11.0" + prisma generate --schema litellm/proxy/schema.prisma + - save_cache: + key: ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }} + paths: + - ~/.local/lib + - ~/.local/bin + - restore_cache: + keys: + - ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - run: + name: Install Node dependencies and Playwright + command: | + cd ui/litellm-dashboard + npm ci + npx playwright install chromium --with-deps + - save_cache: + key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - run: + name: Build UI from source + command: | + cd ui/litellm-dashboard + npm run build + cp -r out/ ../../litellm/proxy/_experimental/out/ + # Restructure HTML so extensionless routes work (login.html -> login/index.html) + find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do + d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" + done + - run: + name: Wait for PostgreSQL + command: dockerize -wait tcp://localhost:5432 -timeout 30s + - run: + name: Push Prisma schema + command: prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Seed database + command: | + PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ + -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql + - run: + name: Start mock LLM server + command: python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py + background: true + - run: + name: Start LiteLLM proxy + environment: + LITELLM_MASTER_KEY: "sk-1234" + MOCK_LLM_URL: "http://127.0.0.1:8090/v1" + DISABLE_SCHEMA_UPDATE: "true" + SERVER_ROOT_PATH: "" + PROXY_LOGOUT_URL: "" + command: | + python -m litellm.proxy.proxy_cli \ + --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ + --port 4000 + background: true + - run: + name: Wait for proxy to be ready + command: | + for i in $(seq 1 60); do + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + echo "Proxy is ready" + exit 0 + fi + sleep 2 + done + echo "Proxy failed to start" + exit 1 + - run: + name: Run Playwright E2E tests + command: | + cd ui/litellm-dashboard + npx playwright test --config e2e_tests/playwright.config.ts + no_output_timeout: 10m + - store_artifacts: + path: ui/litellm-dashboard/test-results + destination: e2e-test-results + - store_artifacts: + path: ui/litellm-dashboard/playwright-report + destination: e2e-playwright-report + build_docker_database_image: machine: image: ubuntu-2204:2024.04.1 @@ -3207,80 +3261,6 @@ jobs: paths: - litellm-docker-database.tar.zst - e2e_ui_testing: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: large - working_directory: ~/project - parameters: - browser: - type: string - steps: - - checkout - - setup_google_dns - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Install Dependencies - command: | - npm install -D @playwright/test - - run: - name: Install Playwright Browsers - command: | - npx playwright install - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e UI_USERNAME="admin" \ - -e UI_PASSWORD="gm" \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - --name litellm-docker-database-<< parameters.browser >> \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start outputting logs - command: docker logs -f litellm-docker-database-<< parameters.browser >> - background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m - - run: - name: Run Playwright Tests - command: | - npx playwright test \ - --project << parameters.browser >> \ - --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ - --reporter=html \ - --output=test-results - no_output_timeout: 15m - - store_artifacts: - path: test-results - destination: playwright-results - - - store_artifacts: - path: playwright-report - destination: playwright-report prisma_schema_sync: machine: @@ -3509,32 +3489,12 @@ workflows: only: - main - /litellm_.*/ - # - e2e_ui_testing: - # name: e2e_ui_testing_chromium - # browser: chromium - # context: e2e_ui_tests - # requires: - # - ui_build - # - build_docker_database_image - # - prisma_schema_sync - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - e2e_ui_testing: - # name: e2e_ui_testing_firefox - # browser: firefox - # context: e2e_ui_tests - # requires: - # - ui_build - # - build_docker_database_image - # - prisma_schema_sync - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ + - e2e_ui_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - build_and_test: requires: - build_docker_database_image @@ -3605,12 +3565,6 @@ workflows: only: - main - /litellm_.*/ - - proxy_e2e_azure_batches_tests: - filters: - branches: - only: - - main - - /litellm_.*/ - llm_translation_testing: filters: branches: @@ -3729,6 +3683,12 @@ workflows: only: - main - /litellm_.*/ + - redis_caching_unit_tests: + filters: + branches: + only: + - main + - /litellm_.*/ - upload-coverage: requires: - realtime_translation_testing @@ -3747,6 +3707,7 @@ workflows: - image_gen_testing - logging_testing - audio_testing + - redis_caching_unit_tests - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d830c16dfa27..210f232b1704 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -32,6 +32,13 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac - [ ] **Merge / cherry-pick CI run** Links: +## Screenshots / Proof of Fix + + + ## Type diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index f1ae30e67d76..d8dec73c4289 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -27,6 +27,10 @@ on: required: false type: number default: 10 + artifact-name: + description: "Unique name for the coverage artifact (must be unique per run)" + required: true + type: string permissions: contents: read @@ -93,4 +97,46 @@ jobs: --reruns "${RERUNS}" \ --reruns-delay 1 \ --dist=loadscope \ - --durations=20 + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml + + - name: Save coverage report + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage.xml + retention-days: 1 + + upload-coverage: + name: Upload coverage to Codecov + needs: run + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + pull-requests: write + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Download coverage report + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage-reports + merge-multiple: true + + - name: Upload to Codecov + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + use_oidc: true + directory: coverage-reports + root_dir: ${{ github.workspace }} + fail_ci_if_error: false diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index d53a9e8822ae..ce4c048c624c 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -27,23 +27,17 @@ on: required: false type: number default: 10 - enable-redis: - description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" - required: false - type: boolean - default: false enable-postgres: description: "Start a local Postgres service container and run Prisma migrations" required: false type: boolean default: false - secrets: - REDIS_HOST: - required: false - REDIS_PORT: - required: false - REDIS_PASSWORD: + artifact-name: + description: "Unique name for the coverage artifact (must be unique per run)" required: false + type: string + default: "run" + secrets: DATABASE_URL: required: false POSTGRES_USER: @@ -61,11 +55,8 @@ jobs: timeout-minutes: ${{ inputs.timeout-minutes }} # Environment is derived from the enable-* flags, not caller-controllable. # This prevents callers from passing arbitrary environment names to bypass secret scoping. - # Note: Postgres service container always starts (GHA limitation), so any Redis job - # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. environment: >- ${{ - inputs.enable-redis && 'integration-redis-postgres' || inputs.enable-postgres && 'integration-postgres' || '' }} @@ -141,9 +132,6 @@ jobs: WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} - REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} - REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} - REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} run: | if [ "${WORKERS}" = "0" ]; then poetry run pytest ${TEST_PATH:?} \ @@ -151,7 +139,10 @@ jobs: --maxfail="${MAX_FAILURES}" \ --reruns "${RERUNS}" \ --reruns-delay 1 \ - --durations=20 + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml else poetry run pytest ${TEST_PATH:?} \ --tb=short -vv \ @@ -160,5 +151,47 @@ jobs: --reruns "${RERUNS}" \ --reruns-delay 1 \ --dist=loadscope \ - --durations=20 + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml fi + + - name: Save coverage report + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage.xml + retention-days: 1 + + upload-coverage: + name: Upload coverage to Codecov + needs: run + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + pull-requests: write + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Download coverage report + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage-reports + merge-multiple: true + + - name: Upload to Codecov + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + use_oidc: true + directory: coverage-reports + root_dir: ${{ github.workspace }} + fail_ci_if_error: false diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 2ae01823a967..b86339798540 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -48,7 +48,21 @@ jobs: const cosignSection = [ `## Verify Docker Image Signature`, ``, - `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`, + `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit \`0112e53\`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).`, + ``, + `**Verify using the pinned commit hash (recommended):**`, + ``, + `A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:`, + ``, + '```bash', + `cosign verify \\`, + ` --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \\`, + ` ghcr.io/berriai/litellm:${tag}`, + '```', + ``, + `**Verify using the release tag (convenience):**`, + ``, + `Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:`, ``, '```bash', `cosign verify \\`, diff --git a/.github/workflows/run_llm_translation_tests.py b/.github/workflows/run_llm_translation_tests.py old mode 100755 new mode 100644 diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml deleted file mode 100644 index dafabe4d83e1..000000000000 --- a/.github/workflows/test-litellm-matrix.yml +++ /dev/null @@ -1,214 +0,0 @@ -name: LiteLLM Unit Tests (Matrix) - -on: - pull_request: - branches: [main] - -permissions: - contents: read - -# Cancel in-progress runs for the same PR -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 20 # Increased from 15 to 20 - strategy: - fail-fast: false - matrix: - test-group: - # tests/test_litellm split by subdirectory (~560 files total) - # Vertex AI tests separated for better isolation (prevent auth/env pollution) - - name: "llms-vertex" - path: "tests/test_litellm/llms/vertex_ai" - workers: 1 - reruns: 2 - - name: "llms-other" - path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" - workers: 2 - reruns: 2 - # tests/test_litellm/proxy split by subdirectory (~180 files total) - - name: "proxy-guardrails" - path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" - workers: 2 - reruns: 2 - - name: "proxy-core" - path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" - workers: 2 - reruns: 2 - - name: "proxy-misc" - path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" - workers: 2 - reruns: 2 - - name: "integrations" - path: "tests/test_litellm/integrations" - workers: 2 - reruns: 3 # Integration tests tend to be flakier - - name: "core-utils" - path: "tests/test_litellm/litellm_core_utils" - workers: 2 - reruns: 1 - - name: "other-1" - # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines - path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" - workers: 2 - reruns: 2 - - name: "other-2" - # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines - path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils" - workers: 2 - reruns: 2 - - name: "other-3" - # remaining dirs ≈ 8.0k lines - path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores" - workers: 2 - reruns: 2 - - name: "root" - path: "tests/test_litellm/test_*.py" - workers: 2 - reruns: 2 - # tests/proxy_unit_tests split alphabetically (~48 files total) - - name: "proxy-unit-a1" - # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest - path: "tests/proxy_unit_tests/test_[a-j]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-a2" - # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback - path: "tests/proxy_unit_tests/test_[k-o]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b1" - # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*) - path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b2" - # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests - path: "tests/proxy_unit_tests/test_proxy_server.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b3" - # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests - path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b4" - # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter - path: "tests/proxy_unit_tests/test_proxy_utils.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b5" - # proxy_token_counter (1279) - runs independently from utils - path: "tests/proxy_unit_tests/test_proxy_token_counter.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b6" - # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62) - path: "tests/proxy_unit_tests/test_[r-t]*.py" - workers: 2 - reruns: 1 - - name: "proxy-unit-b7" - # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157) - path: "tests/proxy_unit_tests/test_[u-z]*.py" - workers: 2 - reruns: 1 - - name: test (${{ matrix.test-group.name }}) - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Install Poetry - run: pip install 'poetry==2.3.2' - - - name: Cache Poetry dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 - with: - path: | - ~/.cache/pypoetry - ~/.cache/pip - .venv - key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry- - - - name: Install dependencies - run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy semantic-router" - # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies - poetry run pip install google-genai==1.22.0 \ - google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache - run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma - - - name: Run tests - ${{ matrix.test-group.name }} - run: | - poetry run pytest ${{ matrix.test-group.path }} \ - --tb=short -vv \ - --maxfail=10 \ - -n ${{ matrix.test-group.workers }} \ - --reruns ${{ matrix.test-group.reruns }} \ - --reruns-delay 1 \ - --dist=loadscope \ - --durations=20 \ - --cov=litellm \ - --cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \ - --cov-config=pyproject.toml - - - name: Save coverage report - if: always() - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: coverage-${{ matrix.test-group.name }} - path: coverage-${{ matrix.test-group.name }}.xml - retention-days: 1 - - upload-coverage: - name: Upload coverage to Codecov - needs: test - if: always() - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write # Required for OIDC tokenless upload - pull-requests: write # Required for Codecov PR comments - - steps: - - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - - name: Download all coverage reports - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 - with: - pattern: coverage-* - path: coverage-reports - merge-multiple: true - - - name: Upload to Codecov - uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 - with: - use_oidc: true - directory: coverage-reports - root_dir: ${{ github.workspace }} - fail_ci_if_error: false diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml deleted file mode 100644 index 7cbbe0b338f0..000000000000 --- a/.github/workflows/test-proxy-e2e-azure-batches.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Proxy E2E Azure Batches Tests - -on: - pull_request: - branches: [main] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - proxy_e2e_azure_batches_tests: - runs-on: ubuntu-latest - timeout-minutes: 30 - - services: - postgres: - image: postgres:15 - env: - POSTGRES_USER: llmproxy - POSTGRES_PASSWORD: dbpassword9090 - POSTGRES_DB: litellm - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Install Poetry - run: pip install 'poetry==2.3.2' - - - name: Cache Poetry dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 - with: - path: | - ~/.cache/pypoetry - ~/.cache/pip - .venv - key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry-e2e-batches- - ${{ runner.os }}-poetry- - - - name: Install dependencies - run: | - poetry config virtualenvs.in-project true - poetry install --with dev,proxy-dev --extras "proxy" - poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4 - - - name: Setup litellm-enterprise - run: | - poetry run pip install --force-reinstall --no-deps -e enterprise/ - - - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache - run: | - poetry run pip install nodejs-wheel-binaries==24.13.1 - poetry run prisma generate --schema litellm/proxy/schema.prisma - - - name: Run Prisma migrations - env: - DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - run: | - cd litellm/proxy - poetry run prisma migrate deploy --schema schema.prisma - cd ../.. - - - name: Run Azure Batch E2E Tests - env: - DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm - USE_LOCAL_LITELLM: "true" - USE_MOCK_MODELS: "true" - USE_STATE_TRACKER: "true" - LITELLM_LOG: DEBUG - run: | - poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \ - -vv -s -k "test_e2e_managed_batch" \ - --tb=short \ - --maxfail=3 \ - --durations=10 diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml deleted file mode 100644 index ca274324f2f2..000000000000 --- a/.github/workflows/test-unit-caching-redis.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: "Unit Tests: Caching (Redis)" - -# Uses cloud Redis credentials — only runs on trusted branches, not PRs. -# This prevents external PRs from accessing Redis credentials. -on: - push: - branches: [main, "litellm_*"] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - caching-redis: - uses: ./.github/workflows/_test-unit-services-base.yml - with: - # Redis-only tests that do NOT require provider API keys. - # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, - # test_router_caching.py) are in Phase 3 integration workflows. - test-path: >- - tests/local_testing/test_dual_cache.py - tests/local_testing/test_redis_batch_optimizations.py - tests/local_testing/test_router_utils.py - workers: 2 - reruns: 2 - timeout-minutes: 20 - enable-redis: true - enable-postgres: false - secrets: - REDIS_HOST: ${{ secrets.REDIS_HOST }} - REDIS_PORT: ${{ secrets.REDIS_PORT }} - REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} - DATABASE_URL: ${{ secrets.DATABASE_URL }} - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml index 2f3698fdf600..9696cea56161 100644 --- a/.github/workflows/test-unit-core-utils.yml +++ b/.github/workflows/test-unit-core-utils.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +20,4 @@ jobs: test-path: "tests/test_litellm/litellm_core_utils" workers: 2 reruns: 1 + artifact-name: core-utils diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml index 13ae3efedba2..986de1195357 100644 --- a/.github/workflows/test-unit-enterprise-routing.yml +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -22,3 +24,4 @@ jobs: tests/test_litellm/router_strategy workers: 2 reruns: 2 + artifact-name: enterprise-routing diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml index 2789f99d81c3..e73c09d6cd8e 100644 --- a/.github/workflows/test-unit-integrations.yml +++ b/.github/workflows/test-unit-integrations.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +20,4 @@ jobs: test-path: "tests/test_litellm/integrations" workers: 2 reruns: 3 + artifact-name: integrations diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml index 6c00272b0c8f..2fb4cf8c1db0 100644 --- a/.github/workflows/test-unit-llm-providers.yml +++ b/.github/workflows/test-unit-llm-providers.yml @@ -14,16 +14,26 @@ concurrency: jobs: vertex-ai: name: Vertex AI + permissions: + contents: read + id-token: write + pull-requests: write uses: ./.github/workflows/_test-unit-base.yml with: test-path: "tests/test_litellm/llms/vertex_ai" workers: 1 reruns: 2 + artifact-name: llm-vertex-ai other-providers: name: All Other Providers + permissions: + contents: read + id-token: write + pull-requests: write uses: ./.github/workflows/_test-unit-base.yml with: test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" workers: 2 reruns: 2 + artifact-name: llm-other-providers diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index 9228decd7cc8..e44133867e64 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -29,3 +31,4 @@ jobs: tests/test_litellm/test_*.py workers: 2 reruns: 2 + artifact-name: misc diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml index e71821db701f..5e427a39f35d 100644 --- a/.github/workflows/test-unit-proxy-auth.yml +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +20,4 @@ jobs: test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" workers: 2 reruns: 2 + artifact-name: proxy-auth diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index bdfb6efeef1c..49d399e87417 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -14,6 +14,10 @@ concurrency: jobs: proxy-db: + permissions: + contents: read + id-token: write + pull-requests: write strategy: fail-fast: false matrix: @@ -37,8 +41,8 @@ jobs: workers: ${{ matrix.workers }} reruns: 2 timeout-minutes: ${{ matrix.timeout }} - enable-redis: false enable-postgres: true + artifact-name: proxy-db-${{ matrix.test-group }} secrets: DATABASE_URL: ${{ secrets.DATABASE_URL }} POSTGRES_USER: ${{ secrets.POSTGRES_USER }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index caff3b3ae061..67d35ef794ef 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -33,3 +35,4 @@ jobs: tests/test_litellm/proxy/ui_crud_endpoints workers: 2 reruns: 2 + artifact-name: proxy-endpoints diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml index 4dfbbe317edd..56801569345a 100644 --- a/.github/workflows/test-unit-proxy-infra.yml +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -26,3 +28,4 @@ jobs: tests/test_litellm/proxy/test_*.py workers: 2 reruns: 2 + artifact-name: proxy-infra diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml index 7f3acac28038..771a695a70c7 100644 --- a/.github/workflows/test-unit-responses-caching-types.yml +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -6,6 +6,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -18,3 +20,4 @@ jobs: test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" workers: 2 reruns: 2 + artifact-name: responses-caching-types diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml index b38c82b1c24b..76d3be3e63c0 100644 --- a/.github/workflows/test-unit-security.yml +++ b/.github/workflows/test-unit-security.yml @@ -7,6 +7,8 @@ on: permissions: contents: read + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -20,8 +22,8 @@ jobs: workers: 1 reruns: 2 timeout-minutes: 20 - enable-redis: false enable-postgres: true + artifact-name: security secrets: DATABASE_URL: ${{ secrets.DATABASE_URL }} POSTGRES_USER: ${{ secrets.POSTGRES_USER }} diff --git a/.trivyignore b/.trivyignore deleted file mode 100644 index 0d04ecacdb56..000000000000 --- a/.trivyignore +++ /dev/null @@ -1,12 +0,0 @@ -# LiteLLM Trivy Ignore File -# CVEs listed here are temporarily allowlisted pending fixes - -# Next.js vulnerabilities in UI dashboard (next@14.2.35) -# Allowlisted: 2026-01-31, 7-day fix timeline -# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+ - -# HIGH: DoS via request deserialization -GHSA-h25m-26qc-wcjf - -# MEDIUM: Image Optimizer DoS -CVE-2025-59471 diff --git a/AGENTS.md b/AGENTS.md index ba9c9b356bc6..37411938e2fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -254,7 +254,7 @@ See `CLAUDE.md` and the `Makefile` for standard commands. Key notes: - `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`. - The `--timeout` pytest flag is NOT available; don't pass it. - Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4` -- Black `--check` may report pre-existing formatting issues; this does not block test runs. +- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI. - If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file. ### Lint diff --git a/CLAUDE.md b/CLAUDE.md index f0478120181a..a8800ff88844 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - `make format` - Apply Black code formatting - `make lint-ruff` - Run Ruff linting only - `make lint-mypy` - Run MyPy type checking only +- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI. ### Single Test Files - `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77bc15ff50b8..c029ccce1ab9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,6 +149,19 @@ Apply formatting (auto-fixes issues): make format ``` +> **Black formatting is enforced in CI.** All PRs must pass the Black formatting check. +> +> - **AI coding agents** (Claude Code, Copilot, Cursor, etc.): `AGENTS.md` and `CLAUDE.md` instruct agents to run `poetry run black .` before committing. +> - **VS Code users**: Install the [Black Formatter extension](https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter) and enable format-on-save: +> ```json +> { +> "[python]": { +> "editor.defaultFormatter": "ms-python.black-formatter", +> "editor.formatOnSave": true +> } +> } +> ``` + ### CI Compatibility To ensure your changes will pass CI, run the exact same checks locally: diff --git a/README.md b/README.md index a6dc597574eb..d7b8bad69f32 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,24 @@ 🚅 LiteLLM
-
Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.] +
LiteLLM AI Gateway
+Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.
-+AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token", +AICORE_CLIENT_ID = " *** ", +AICORE_CLIENT_SECRET = " *** ", +AICORE_RESOURCE_GROUP = " *** ", +AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2" ++ +Other credential configuration options are also available. For more information, see the [SAP AI Core Documentation](https://help.sap.com/doc/generative-ai-hub-sdk/CLOUD/en-US/_reference/README_sphynx.html#configuration). +## Usage - LiteLLM Python SDK ### Proxy Usage @@ -506,6 +526,241 @@ response = embedding( print(response.data[0]["embedding"]) # Vector representation ``` +### Additional Modules +The SAP Gen AI Hub includes additional modules for advanced use cases: +- [Grounding](https://help.sap.com/docs/sap-ai-core/generative-ai/grounding-035c455a5a424697b60f4a24b6d791fe?locale=en-US) +- [Translation](https://help.sap.com/docs/sap-ai-core/generative-ai/translation?locale=en-US) +- [Data Masking](https://help.sap.com/docs/sap-ai-core/generative-ai/data-masking-d9a54d9ca54b40beacbd24e1663ec3b4?locale=en-US) +- [Content Filtering](https://help.sap.com/docs/sap-ai-core/generative-ai/content-filtering?locale=en-US) + +#### Grounding +Grounding is a service designed to handle data-related tasks, such as grounding and retrieval, using vector databases. It provides specialized data retrieval through these databases, grounding the retrieval process with your own external and context-relevant data. Grounding combines generative AI capabilities with the ability to use real-time, precise data to improve decision-making and business operations for specific AI-driven business solutions. +##### Prerequisites +To use the Grounding module in the orchestration pipeline, you need to prepare the knowledge base in advance. + +Generative AI hub offers multiple options for users to provide data (prepare a knowledge base): +- For Option 1: Upload the documents to a supported data repository and run the data pipeline to vectorize the documents. +- For Option 2: Provide the chunks of document via Vector API directly. + +To use grounding, choose from one of the following options. + +Usage example: +```python showLineNumbers title="Grounding Example" +from litellm import completion + +grounding_config = { + 'type': 'document_grounding_service', + 'config': { + 'filters': [ + {'id': 's3-docs', + 'data_repository_type': 'vector', + 'search_config': {'max_chunk_count': 2}, + 'data_repositories': ['012345-6789-0123-4567-890123456789'] + } + ], + 'placeholders': {'input': ['user_query'], 'output': 'grounding_response'}, + 'metadata_params': ['source', 'webUrl', 'title', 'mimeType', 'fileSuffix'] + } +} + +response = completion(model="sap/gpt-4o", + messages=[ + {"content":"""Facility Solutions Company provides services to luxury residential complexes, + apartments, individual homes, and commercial properties such as office buildings, retail + spaces, industrial facilities, and educational institutions. Customers are encouraged to + reach out with maintenance requests, service deficiencies, follow-ups, or any issues they + need by email.""", "role": "system"}, + {"content":"""You are a helpful assistant for any queries for answering questions. + Answer the request by providing relevant answers that fit to the request. + Request: {{ ?user_query }} + Context:{{ ?grounding_response }}""", "role": "user"} + ], + placeholder_values={"user_query": "Is there a complaint?"}, + grounding=grounding_config + ) +print(response.choices[0].message.content) +``` +For more information about all available grounding configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/using-grounding-module-e1c4dd100dfb42ab890e1d95f3516187?locale=en-US). + +#### Translation +The translation module allows you to translate LLM text prompts into a chosen target language. + +```python showLineNumbers title="Translation Example" +from litellm import completion + +translation_config = { + 'input': + {'type': 'sap_document_translation', + 'config': + {'source_language': 'en-US', + 'target_language': 'de-DE'} + }, + 'output': + {'type': 'sap_document_translation', + 'config': + {'source_language': 'de-DE', + 'target_language': 'fr-FR'} + } +} + +response = completion(model="sap/gpt-4o", + messages=[{"role": "user", "content": "Hello world!"}], + translation=translation_config) + +print(response.choices[0].message.content) +``` +For more information about all available translation configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/translation?locale=en-US) + +#### Data Masking +The data masking module serves to anonymize or pseudonymize personally identifiable information from the input for selected entities. + +```python showLineNumbers title="Data Masking Example" +from litellm import completion, embedding +masking_config = { + 'providers': + [ + { + 'type': 'sap_data_privacy_integration', + 'method': 'anonymization', + 'entities': [ + {'type': 'profile-address'}, + {'type': 'profile-email'}, + {'type': 'profile-phone'}, + {'type': 'profile-person'}, + {'type': 'profile-location'} + ] + } + ] + } + +mock_cv = "some text with personal information" + +response = completion(model="sap/gpt-4o", + messages=[{"role": "user", "content": "Give a one sentence summary of the CV. CV: {{?cv}}?"}], + placeholder_values={"cv": mock_cv}, + masking=masking_config) +print(response.choices[0].message.content) + +# Data masking module also available for embedding +response = embedding(model="sap/text-embedding-3-small", + input=mock_cv, + masking=masking_config) +print(response.data[0]) +``` +For more information about all available data masking configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/enhancing-model-consumption-with-data-masking-66ad6f469afc4c2cbaa91a27a33f7b21?locale=en-US) + + + + + +#### Content Filtering +The content filtering module allows you to filter input and output based on content safety criteria. + +The module supports two services: +* Azure Content Safety +* Llama Guard 3 + +```python showLineNumbers title="Content Filtering Example" +from litellm import completion + +filtering_config_azure = { + 'input': + { + 'filters': + [ + {'type': 'azure_content_safety', + 'config': + {'hate': 0, + 'sexual': 0, + 'violence': 0, + 'self_harm': 0 + } + } + ] + }, + 'output': + { + 'filters': + [ + {'type': 'azure_content_safety', + 'config': {'hate': 0, + 'sexual': 0, + 'violence': 0, + 'self_harm': 0 + } + } + ] + } +} + +response = completion(model="sap/gpt-4o", + messages=[{"role": "user", "content": "Hello world!"}], + filtering=filtering_config_azure) +print(response.choices[0].message.content) +# The model responds normally because the content does not violate any safety rules. + +try: + response = completion(model="sap/gpt-4o", + messages=[{"role": "user", "content": "I hate you"}], + filtering=filtering_config_azure) +except Exception as e: + print(e) + # The service raises an error: + # "Input Filter: Content filtered due to safety violations. Please modify the prompt and try again." +``` +For more information about all available content filtering configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/content-filtering?locale=en-US) + +#### List of modules configuration for fallback +SAP GEN AI Hub supports a fallback mechanism for handling errors. This mechanism allows you to specify a list of fallback modules to use in case of errors. The fallback modules should contain all parameters that are required for configuring the request. + +Required parameters: +- `model` +- `messages` + +Optional parameters: +- `filtering` +- `grounding` +- `translation` +- `masking` +- `tools` + +- and any of model's specific parameters. + + +```python showLineNumbers title="Fallback Example" +from litellm import completion + +translation_config = { + 'input': + {'type': 'sap_document_translation', + 'config': + {'source_language': 'en-US', + 'target_language': 'de-DE'} + }, + 'output': + {'type': 'sap_document_translation', + 'config': + {'source_language': 'de-DE', + 'target_language': 'fr-FR'} + } +} + +response = completion(model="sap/gpt-4o", + messages=[{"role": "user", "content": "Hello world!"}], + translation=translation_config, + fallback_sap_modules=[{ + "model":"sap/gemini-2.5-flash", + "messages":[{"role": "user", "content": "Hello world!"}], + "translation":translation_config + }]) + +# In case of error with the first configuration (model gpt-4o), the fallback module is used. + +print(response.choices[0].message.content) + +``` + + ## Reference ### Supported Parameters diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 528a5c109035..db38cf5426bb 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -197,6 +197,7 @@ router_settings: | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) | | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. | | use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. | +| skip_system_message_in_guardrail | boolean | If true, unified guardrails omit `role: system` from scanned input on **chat completions** and **Anthropic `/v1/messages`** only; the LLM still receives full messages. Per-guardrail override: `litellm_params.skip_system_message_in_guardrail` on each guardrail. [Guardrails quick start](./guardrails/quick_start#skip-system-messages-in-guardrail-evaluation) | | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). | | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. | | enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. | @@ -238,7 +239,7 @@ router_settings: | public_routes | List[str] | (Enterprise Feature) Control list of public routes | | alert_types | List[str] | Control list of alert types to send to slack (Doc on alert types)[./alerting.md] | | enforced_params | List[str] | (Enterprise Feature) List of params that must be included in all requests to the proxy | -| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication | +| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication on LLM + info routes | | use_x_forwarded_for | str | If true, uses the X-Forwarded-For header to get the client IP address | | service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] | | image_generation_model | str | The default model to use for image generation - ignores model set in request | @@ -597,10 +598,13 @@ router_settings: | LITELLM_MCP_TOOL_LISTING_TIMEOUT | Timeout in seconds for listing tools from an MCP server. Default is 30 | LITELLM_MCP_METADATA_TIMEOUT | HTTP client timeout in seconds for OAuth metadata fetching. Default is 10 | LITELLM_MCP_HEALTH_CHECK_TIMEOUT | Health check timeout in seconds for MCP servers. Default is 10 +| LITELLM_MCP_STDIO_EXTRA_COMMANDS | Comma-separated extra command basenames allowed for MCP stdio transport beyond the built-in allowlist. Example: `my-mcp-bin`. Empty by default | MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600 | MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200 | MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10 | MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from token expiry when computing cache TTL. Default is 60 +| MCP_PER_USER_TOKEN_DEFAULT_TTL | Default TTL in seconds for per-user MCP OAuth tokens stored in Redis. Default is 43200 (12 hours) +| MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from per-user MCP OAuth token expiry when computing Redis TTL. Default is 60 | DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 | DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 | DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 @@ -1032,6 +1036,7 @@ router_settings: | SENDGRID_SENDER_EMAIL | Email address used as the sender in SendGrid email transactions | SPEND_LOGS_URL | URL for retrieving spend logs | SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 +| STALE_OBJECT_CLEANUP_BATCH_SIZE | Max number of stale managed objects updated per cleanup cycle. Default is 1000 | SSL_CERTIFICATE | Path to the SSL certificate file | SSL_ECDH_CURVE | ECDH curve for SSL/TLS key exchange (e.g., 'X25519' to disable PQC). | SSL_SECURITY_LEVEL | [BETA] Security level for SSL/TLS connections. E.g. `DEFAULT@SECLEVEL=1` diff --git a/docs/my-website/docs/proxy/credential_routing.md b/docs/my-website/docs/proxy/credential_routing.md new file mode 100644 index 000000000000..2af57c6b4969 --- /dev/null +++ b/docs/my-website/docs/proxy/credential_routing.md @@ -0,0 +1,274 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Per-Team/Project Credential Routing + +Route the same model to different LLM provider endpoints (e.g. different Azure instances) based on which team or project makes the request. + +## Overview + +In multi-tenant deployments, different teams often need the same model name (e.g., `gpt-4`) to hit different provider endpoints — for example, separate Azure OpenAI instances per business unit for cost isolation, data residency, or rate limit separation. + +**Credential routing** lets you configure this in team/project metadata using the existing [credentials table](./ui_credentials.md), without duplicating model definitions or creating separate model groups per team. + +``` +Hotel Team → gpt-4 → https://hotel-eastus.openai.azure.com/ +Flight Team → gpt-4 → https://flight-centralus.openai.azure.com/ +``` + +### Precedence Chain + +When a request comes in, the system walks this precedence chain (first match wins): + +1. **Clientside credentials** — `api_base`/`api_key` passed in the request body ([docs](./clientside_auth.md)) +2. **Project model-specific** — override for this exact model in the project's `model_config` +3. **Project default** — `defaultconfig` in the project's `model_config` +4. **Team model-specific** — override for this exact model in the team's `model_config` +5. **Team default** — `defaultconfig` in the team's `model_config` +6. **Deployment default** — the model's `litellm_params` as configured in `config.yaml` + +## Quick Start + +### Step 1: Create Credentials + +Store your Azure endpoint credentials in the credentials table. You can do this via the [UI](./ui_credentials.md) or API: + +```bash showLineNumbers +# Create credential for Hotel team's Azure endpoint +curl -X POST 'http://0.0.0.0:4000/credentials' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "credential_name": "hotel-azure-eastus", + "credential_values": { + "api_base": "https://hotel-eastus.openai.azure.com/", + "api_key": "sk-azure-hotel-key-xxx" + } +}' +``` + +```bash showLineNumbers +# Create credential for Flight team's Azure endpoint +curl -X POST 'http://0.0.0.0:4000/credentials' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "credential_name": "flight-azure-centralus", + "credential_values": { + "api_base": "https://flight-centralus.openai.azure.com/", + "api_key": "sk-azure-flight-key-xxx" + } +}' +``` + +### Step 2: Set `model_config` on Teams + +Add a `model_config` key to the team's metadata referencing the credential by name: + +```bash showLineNumbers +# Hotel team — default Azure endpoint for all models +curl -X PATCH 'http://0.0.0.0:4000/team/update' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "team_id": "hotel-team-id", + "metadata": { + "model_config": { + "defaultconfig": { + "azure": { + "litellm_credentials": "hotel-azure-eastus" + } + } + } + } +}' +``` + +```bash showLineNumbers +# Flight team — default Azure endpoint for all models +curl -X PATCH 'http://0.0.0.0:4000/team/update' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "team_id": "flight-team-id", + "metadata": { + "model_config": { + "defaultconfig": { + "azure": { + "litellm_credentials": "flight-azure-centralus" + } + } + } + } +}' +``` + +### Step 3: Make Requests + +Requests are automatically routed to the correct Azure endpoint based on the API key's team: + +```bash showLineNumbers +# Request using Hotel team's API key → routes to hotel-eastus.openai.azure.com +curl http://localhost:4000/v1/chat/completions \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-hotel-team-key' \ +-d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' + +# Request using Flight team's API key → routes to flight-centralus.openai.azure.com +curl http://localhost:4000/v1/chat/completions \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-flight-team-key' \ +-d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +## Per-Model Overrides + +You can set different credentials for specific models while keeping a default for everything else: + +```bash showLineNumbers +curl -X PATCH 'http://0.0.0.0:4000/team/update' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "team_id": "hotel-team-id", + "metadata": { + "model_config": { + "defaultconfig": { + "azure": { + "litellm_credentials": "hotel-azure-eastus" + } + }, + "gpt-4": { + "azure": { + "litellm_credentials": "hotel-azure-westus" + } + } + } + } +}' +``` + +With this config: +- `gpt-4` requests → `hotel-azure-westus` credential (model-specific) +- All other models → `hotel-azure-eastus` credential (default) + +## Project-Level Overrides + +Projects inherit their team's `model_config` but can override at the project level. Project overrides take precedence over team overrides. + +```bash showLineNumbers +# Project overrides the team default for all models +curl -X PATCH 'http://0.0.0.0:4000/project/update' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "project_id": "hotel-rec-app-id", + "metadata": { + "model_config": { + "defaultconfig": { + "azure": { + "litellm_credentials": "hotel-rec-azure" + } + }, + "gpt-4-vision": { + "azure": { + "litellm_credentials": "hotel-rec-vision" + } + } + } + } +}' +``` + +### Full Example: Hotel Team with Two Projects + +**Setup:** +- **Hotel Team**: default `hotel-azure-eastus`, GPT-4 override to `hotel-azure-westus` +- **Hotel Rec App** (project): default `hotel-rec-azure`, GPT-4-Vision override to `hotel-rec-vision` +- **Hotel Review App** (project): no overrides — inherits team config + +**Resolution:** + +| Request | Resolved Credential | Why | +|---|---|---| +| Hotel Rec App → `gpt-4` | `hotel-rec-azure` | Project default (no project model-specific match for gpt-4) | +| Hotel Rec App → `gpt-4-vision` | `hotel-rec-vision` | Project model-specific | +| Hotel Review App → `gpt-3.5` | `hotel-azure-eastus` | Team default (no project config) | +| Hotel Review App → `gpt-4` | `hotel-azure-westus` | Team model-specific | + +## `model_config` Schema + +The `model_config` key is a JSON object in team/project `metadata`: + +```json +{ + "model_config": { + "defaultconfig": { + "