Fix json_tool_call filtering: streaming support, pop->get, behavioral parity

.circleci/config.yml

@@ -1255,7 +1255,15 @@ jobs:
             ls
             # Add --timeout to kill hanging tests after 120s (2 min)
             # Add --durations=20 to show 20 slowest tests for debugging
-            python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
+            # Subdirectories with dedicated jobs (maintain this list as new jobs are added)
+            IGNORE_DIRS=(
+              "tests/llm_translation/realtime"
+            )
+            IGNORE_ARGS=""
+            for dir in "${IGNORE_DIRS[@]}"; do
+              IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
+            done
+            python -m pytest -vv tests/llm_translation $IGNORE_ARGS --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -1271,6 +1279,54 @@ jobs:
           paths:
             - llm_translation_coverage.xml
             - llm_translation_coverage
+  realtime_translation_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.22.0"
+            pip install "pytest-xdist==3.6.1"
+            pip install "pytest-timeout==2.2.0"
+            pip install "websockets"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run realtime tests
+          command: |
+            pwd
+            ls
+            # Add --timeout to kill hanging tests after 120s (2 min)
+            # Add --durations=20 to show 20 slowest tests for debugging
+            python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml realtime_translation_coverage.xml
+            mv .coverage realtime_translation_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - realtime_translation_coverage.xml
+            - realtime_translation_coverage
   mcp_testing:
     docker:
       - image: cimg/python:3.11
@@ -1316,6 +1372,51 @@ jobs:
           paths:
             - mcp_coverage.xml
             - mcp_coverage
+  agent_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.22.0"
+            pip install "pydantic==2.11.0"
+            pip install "a2a-sdk"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml agent_coverage.xml
+            mv .coverage agent_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - agent_coverage.xml
+            - agent_coverage
   guardrails_testing:
     docker:
       - image: cimg/python:3.11
@@ -3532,7 +3633,7 @@ jobs:
             python -m venv venv
             . venv/bin/activate
             pip install coverage
-            coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
+            coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
             coverage xml
       - codecov/upload:
           file: ./coverage.xml
@@ -3754,6 +3855,9 @@ jobs:
 
             cd ui/litellm-dashboard
 
+            # Remove node_modules and package-lock to ensure clean install (fixes dependency resolution issues)
+            rm -rf node_modules package-lock.json
+
             # Install dependencies first
             npm install
 
@@ -4193,12 +4297,24 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - realtime_translation_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - mcp_testing:
           filters:
             branches:
               only:
                 - main
                 - /litellm_.*/
+      - agent_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - guardrails_testing:
           filters:
             branches:
@@ -4304,7 +4420,9 @@ workflows:
       - upload-coverage:
           requires:
             - llm_translation_testing
+            - realtime_translation_testing
             - mcp_testing
+            - agent_testing
             - google_generate_content_endpoint_testing
             - guardrails_testing
             - llm_responses_api_testing
@@ -4381,7 +4499,9 @@ workflows:
             - e2e_openai_endpoints
             - test_bad_database_url
             - llm_translation_testing
+            - realtime_translation_testing
             - mcp_testing
+            - agent_testing
             - google_generate_content_endpoint_testing
             - llm_responses_api_testing
             - ocr_testing

CLAUDE.md

@@ -90,6 +90,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Pydantic v2 for data validation
 - Async/await patterns throughout
 - Type hints required for all public APIs
+- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
 
 ### Testing Strategy
 - Unit tests in `tests/test_litellm/`

Makefile

@@ -1,7 +1,10 @@
 # LiteLLM Makefile
 # Simple Makefile for running tests and basic development tasks
 
-.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
+.PHONY: help test test-unit test-integration test-unit-helm \
+	info lint lint-dev format \
+	install-dev install-proxy-dev install-test-deps \
+	install-helm-unittest check-circular-imports check-import-safety
 
 # Default target
 help:
@@ -25,6 +28,13 @@ help:
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 
+# Keep PIP simple for edge cases:
+PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
+
+# Show info
+info:
+	@echo "PIP: $(PIP)"
+
 # Installation targets
 install-dev:
 	poetry install --with dev
@@ -34,19 +44,19 @@ install-proxy-dev:
 
 # CI-compatible installations (matches GitHub workflows exactly)
 install-dev-ci:
-	pip install openai==2.8.0
+	$(PIP) install openai==2.8.0
 	poetry install --with dev
-	pip install openai==2.8.0
+	$(PIP) install openai==2.8.0
 
 install-proxy-dev-ci:
 	poetry install --with dev,proxy-dev --extras proxy
-	pip install openai==2.8.0
+	$(PIP) install openai==2.8.0
 
 install-test-deps: install-proxy-dev
-	poetry run pip install "pytest-retry==1.6.3"
-	poetry run pip install pytest-xdist
-	poetry run pip install openapi-core
-	cd enterprise && poetry run pip install -e . && cd ..
+	poetry run $(PIP) install "pytest-retry==1.6.3"
+	poetry run $(PIP) install pytest-xdist
+	poetry run $(PIP) install openapi-core
+	cd enterprise && poetry run $(PIP) install -e . && cd ..
 
 install-helm-unittest:
 	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
@@ -62,8 +72,40 @@ format-check: install-dev
 lint-ruff: install-dev
 	cd litellm && poetry run ruff check . && cd ..
 
+# faster linter for developing ...
+# inspiration from:
+# https://github.com/astral-sh/ruff/discussions/10977
+# https://github.com/astral-sh/ruff/discussions/4049
+lint-format-changed: install-dev
+	@git diff origin/main --unified=0 --no-color -- '*.py' | \
+	perl -ne '\
+		if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
+		if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
+			$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
+			print "$$file:$$start:1-$$end:999\n"; \
+		}' | \
+	while read range; do \
+		file="$${range%%:*}"; \
+		lines="$${range#*:}"; \
+		echo "Formatting $$file (lines $$lines)"; \
+		poetry run ruff format --range "$$lines" "$$file"; \
+	done
+
+lint-ruff-dev: install-dev
+	@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
+	cd litellm && \
+	(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
+	poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
+	cd .. ; \
+	rm -f "$$tmpfile"
+
+lint-ruff-FULL-dev: install-dev
+	@files=$$(git diff --name-only origin/main -- '*.py'); \
+	if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
+	else echo "No changed .py files to check."; fi
+
 lint-mypy: install-dev
-	poetry run pip install types-requests types-setuptools types-redis types-PyYAML
+	poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
 	cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
 
 lint-black: format-check
@@ -72,11 +114,14 @@ check-circular-imports: install-dev
 	cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
 
 check-import-safety: install-dev
-	poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+	@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
 # Combined linting (matches test-linting.yml workflow)
 lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
 
+# Faster linting for local development (only checks changed code)
+lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
+
 # Testing targets
 test:
 	poetry run pytest tests/

ci_cd/security_scans.sh

@@ -154,6 +154,7 @@ run_grype_scans() {
         "CVE-2025-15367" # No fix available yet
         "CVE-2025-12781" # No fix available yet
         "CVE-2025-11468" # No fix available yet
+        "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
     )
 
     # Build JSON array of allowlisted CVE IDs for jq