Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 24 additions & 100 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -164,25 +164,6 @@ jobs:
command: |
uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v

semgrep:
docker:
- image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: medium
steps:
- checkout
- setup_google_dns
- install_uv
- run:
name: Run Semgrep (custom rules only)
command: |
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error

local_testing_part1:
docker:
- image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
Expand Down Expand Up @@ -1283,6 +1264,30 @@ jobs:
ls
uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"

installing_litellm_on_python_3_13:
docker:
- image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: medium

steps:
- checkout
- setup_google_dns
- install_uv
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.13
- run:
name: Run tests
command: |
pwd
ls
uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"

installing_litellm_on_python_v2_migration_resolver:
docker:
- image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
Expand Down Expand Up @@ -1316,29 +1321,6 @@ jobs:
uv run --no-sync python -m pytest -vv \
tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver

installing_litellm_on_python_3_13:
docker:
- image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: medium

steps:
- checkout
- setup_google_dns
- install_uv
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.13
- run:
name: Run tests
command: |
pwd
ls
uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
helm_chart_testing:
machine:
image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker
Expand Down Expand Up @@ -1419,52 +1401,6 @@ jobs:
kind delete cluster --name litellm-test
when: always # This ensures cleanup runs even if previous steps fail

check_code_and_doc_quality:
docker:
- image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project/litellm

steps:
- checkout
- setup_google_dns
- install_uv
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- run: uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
- run: uv run --no-sync ruff check ./litellm
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
- run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
- run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
- run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
- run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
- run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
- run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
- run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
- run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
- run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
- run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
- run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
- run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
- run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
- run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
- run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
- run: uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
- run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
- run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
# helm lint is handled by the dedicated helm_chart_testing job

db_migration_disable_update_check:
machine:
image: ubuntu-2204:2024.04.1
Expand Down Expand Up @@ -2603,12 +2539,6 @@ workflows:
only:
- main
- /litellm_.*/
- semgrep:
filters:
branches:
only:
- main
- /litellm_.*/
- local_testing_part1:
filters:
branches:
Expand Down Expand Up @@ -2645,12 +2575,6 @@ workflows:
only:
- main
- /litellm_.*/
- check_code_and_doc_quality:
filters:
branches:
only:
- main
- /litellm_.*/
- ui_build:
filters:
branches:
Expand Down
136 changes: 136 additions & 0 deletions .github/workflows/test-code-quality.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
name: Code Quality Checks

on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
code-quality:
runs-on: ubuntu-latest
timeout-minutes: 15

steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false

- name: Checkout litellm-docs (for documentation_tests)
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
repository: BerriAI/litellm-docs
path: _litellm_docs_checkout
persist-credentials: false

- name: Wire up docs path expected by documentation_tests/*
run: |
# documentation_tests scripts read from docs/my-website/docs/...
Comment on lines +33 to +37

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Cross-repo checkout may fail for fork PRs if litellm-docs is private

actions/checkout without an explicit token uses the job's GITHUB_TOKEN, which is scoped to the current repository only. If BerriAI/litellm-docs is a private repository, this step will fail with a 403 for every PR — including those opened by team members — because GITHUB_TOKEN does not grant cross-repo access. Add a PAT (or a fine-grained token stored as a secret) via the token: parameter, or confirm the repo is public so anonymous clone works without credentials.

      - name: Checkout litellm-docs (for documentation_tests)
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          repository: BerriAI/litellm-docs
          path: _litellm_docs_checkout
          persist-credentials: false
          token: ${{ secrets.LITELLM_DOCS_READ_TOKEN }}  # needed if repo is private

# In litellm-docs the same files live at docs/... (repo root).
# Point docs/my-website -> litellm-docs checkout so the paths resolve.
rm -rf docs/my-website
ln -s ../_litellm_docs_checkout docs/my-website

- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"

- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"

- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-

- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras

- name: check_licenses
run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py

- name: check_provider_folders_documented
run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py

- name: router_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py

- name: test_chat_completion_imports
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py

- name: info_log_check
run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py

- name: check_guardrail_apply_decorator
run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py

- name: test_ban_set_verbose
run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py

- name: code_qa_check_tests
run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py

- name: check_get_model_cost_key_performance
run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py

- name: test_proxy_types_import
run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py

- name: callback_manager_test
run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py

- name: recursive_detector
run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py

- name: test_router_strategy_async
run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py

- name: litellm_logging_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py

- name: ensure_async_clients_test
run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py

- name: enforce_llms_folder_style
run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py

- name: prevent_key_leaks_in_exceptions
run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py

- name: check_unsafe_enterprise_import
run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py

- name: ban_copy_deepcopy_kwargs
run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py

- name: check_fastuuid_usage
run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py

- name: memory_test
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py

- name: documentation_test_env_keys
run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py

- name: documentation_test_router_settings
run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py

- name: documentation_test_api_docs
run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
39 changes: 39 additions & 0 deletions .github/workflows/test-semgrep.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Semgrep

on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
semgrep:
runs-on: ubuntu-latest
timeout-minutes: 10

steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false

- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"

- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"

- name: Run Semgrep (custom rules)
run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error
Loading