feat(proxy): add TypeSafe AI Jev evaluate passthrough with registry-priced spend tracking

gateway/routes/allowlist.py

@@ -96,6 +96,7 @@
     "/langfuse/",
     "/vllm/",
     "/mistral/",
+    "/typesafe/",
     "/nvidia_nim/",
     "/groq/",
     "/voyage/",

litellm/model_prices_and_context_window_backup.json

@@ -69184,6 +69184,27 @@
         "supports_reasoning": true,
         "supports_vision": true
     },
+    "typesafe/jev-1.13.0": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-latest": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-preview": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
     "wandb/zai-org/GLM-5.3-Flash": {
         "cache_read_input_token_cost": 5e-08,
         "input_cost_per_token": 1.5e-07,

litellm/proxy/_lazy_features.py

@@ -208,6 +208,7 @@ def matches(self, path: str) -> bool:
             "/nvidia_nim/",
             "/openai/",
             "/openai_passthrough/",
+            "/typesafe/",
             "/vertex-ai/",
             "/vertex_ai/",
             "/vllm/",

litellm/proxy/_lazy_openapi_snapshot.json

@@ -20373,6 +20373,96 @@
           ]
         }
       },
+      "/typesafe/{endpoint}": {
+        "get": {
+          "description": "[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)",
+          "operationId": "typesafe_proxy_route_typesafe__endpoint__get",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "endpoint",
+              "required": true,
+              "schema": {
+                "title": "Endpoint",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Typesafe Proxy Route",
+          "tags": [
+            "llm_passthrough"
+          ]
+        },
+        "post": {
+          "description": "[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)",
+          "operationId": "typesafe_proxy_route_typesafe__endpoint__post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "endpoint",
+              "required": true,
+              "schema": {
+                "title": "Endpoint",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Typesafe Proxy Route",
+          "tags": [
+            "llm_passthrough"
+          ]
+        }
+      },
       "/vertex_ai/discovery/{endpoint}": {
         "delete": {
           "description": "Call any vertex discovery endpoint using the proxy.\n\nJust use `{PROXY_BASE_URL}/vertex_ai/discovery/{endpoint:path}`\n\nTarget url: `https://discoveryengine.googleapis.com`",

litellm/proxy/_types.py

@@ -483,6 +483,7 @@ class LiteLLMRoutes(enum.Enum):
         "/eu.assemblyai",
         "/vllm",
         "/mistral",
+        "/typesafe",
         "/milvus",
         "/gigachat",
         "/watsonx",

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -525,6 +525,42 @@ async def mistral_proxy_route(
     return received_value
 
 
+@router.api_route(
+    "/typesafe/{endpoint:path}",
+    methods=["GET", "POST"],  # mutable-ok: FastAPI route metadata requires a list
+    tags=["TypeSafe AI Pass-through", "pass-through"],  # mutable-ok: FastAPI route metadata requires a list
+)
+async def typesafe_proxy_route(
+    endpoint: str,
+    request: Request,
+    fastapi_response: Response,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+):
+    """[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)"""
+    base_target_url: Final = get_secret_str("TYPESAFE_API_BASE") or "https://api.typesafe.ai"
+    encoded_endpoint: Final = httpx.URL(endpoint).path
+    normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
+    base_url: Final = httpx.URL(base_target_url)
+    updated_url: Final = base_url.copy_with(
+        path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
+    )
+    typesafe_api_key: Final = passthrough_endpoint_router.get_credentials(
+        custom_llm_provider="typesafe",
+        region_name=None,
+    )
+    endpoint_func: Final = create_pass_through_route(
+        endpoint=endpoint,
+        target=str(updated_url),
+        custom_headers={  # mutable-ok: pass-through request headers require a mutable mapping
+            "Authorization": f"Bearer {typesafe_api_key}",
+            "Content-Type": "application/json",
+        },
+        custom_llm_provider="typesafe",
+        is_streaming_request=False,
+    )
+    return await endpoint_func(request, fastapi_response, user_api_key_dict)
+
+
 @router.api_route(
     "/milvus/{endpoint:path}",
     methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py

@@ -0,0 +1,117 @@
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Final
+
+import httpx
+from pydantic import BaseModel, TypeAdapter, ValidationError
+
+import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.litellm_logging import (
+    get_standard_logging_object_payload,  # pyright: ignore[reportUnknownVariableType]  # legacy helper has an untyped signature
+)
+from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
+from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
+
+
+class _TypeSafeUsage(BaseModel):
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+class _TypeSafeResponse(BaseModel):
+    model: str | None = None
+    usage: _TypeSafeUsage | None = None
+
+
+class _RegistryPricing(BaseModel):
+    input_cost_per_token: float = 0.0
+    output_cost_per_token: float = 0.0
+
+
+_TYPESAFE_RESPONSE_ADAPTER: Final = TypeAdapter(_TypeSafeResponse)
+_REGISTRY_PRICING_ADAPTER: Final = TypeAdapter(_RegistryPricing)
+
+
+def _parse_typesafe_response(response_body: Mapping[str, object]) -> _TypeSafeResponse:
+    try:
+        return _TYPESAFE_RESPONSE_ADAPTER.validate_python(response_body)
+    except ValidationError:
+        return _TypeSafeResponse()
+
+
+def _pricing_for(model_keys: tuple[str, ...]) -> _RegistryPricing:
+    for model_key in model_keys:
+        if model_key not in litellm.model_cost:  # pyright: ignore[reportUnknownMemberType]  # registry is dynamically typed
+            continue
+        try:
+            return _REGISTRY_PRICING_ADAPTER.validate_python(
+                litellm.model_cost[model_key]  # pyright: ignore[reportUnknownMemberType]  # registry is dynamically typed
+            )
+        except ValidationError:
+            continue
+    return _RegistryPricing()
+
+
+class TypeSafePassthroughLoggingHandler:
+    @staticmethod
+    def typesafe_passthrough_handler(
+        httpx_response: httpx.Response,
+        response_body: Mapping[str, object],
+        logging_obj: LiteLLMLoggingObj,
+        url_route: str,
+        result: str,
+        start_time: datetime,
+        end_time: datetime,
+        cache_hit: bool,
+        request_body: Mapping[str, object],
+        **kwargs: object,
+    ) -> PassThroughEndpointLoggingTypedDict:
+        response: Final = _parse_typesafe_response(response_body)
+        response_model: Final = response.model
+        request_model_value: Final = request_body.get("model")
+        request_model: Final = request_model_value if isinstance(request_model_value, str) else None
+        logged_model: Final = response_model or request_model or "unknown"
+        model_name: Final = f"typesafe/{logged_model}"
+        usage: Final = response.usage or _TypeSafeUsage()
+        input_tokens: Final = usage.input_tokens
+        output_tokens: Final = usage.output_tokens
+        candidate_model_keys: Final = tuple(
+            f"typesafe/{model}" for model in (response_model, request_model) if model is not None
+        )
+        pricing: Final = _pricing_for(candidate_model_keys)
+        response_cost: Final = (
+            input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
+        )
+        usage_object: Final = Usage(
+            prompt_tokens=input_tokens,
+            completion_tokens=output_tokens,
+            total_tokens=input_tokens + output_tokens,
+        )
+        updated_kwargs: Final = {  # mutable-ok: pass-through logging contract requires mutable kwargs
+            **kwargs,
+            "model": model_name,
+            "custom_llm_provider": "typesafe",
+            "response_cost": response_cost,
+            "combined_usage_object": usage_object,
+        }
+        logging_obj.model_call_details.update(
+            model=model_name,
+            custom_llm_provider="typesafe",
+            response_cost=response_cost,
+        )
+        standard_logging_object: Final = get_standard_logging_object_payload(
+            kwargs=updated_kwargs,
+            init_response_obj=ModelResponse(model=model_name, usage=usage_object),
+            start_time=start_time,
+            end_time=end_time,
+            logging_obj=logging_obj,
+            status="success",
+        )
+        return {  # mutable-ok: pass-through logging contract requires mutable result
+            "result": StandardPassThroughResponseObject(response=result),
+            "kwargs": {  # mutable-ok: pass-through logging contract requires mutable kwargs
+                **updated_kwargs,
+                "standard_logging_object": standard_logging_object,
+            },
+        }

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -1,5 +1,6 @@
 import json
 from datetime import datetime
+from types import MappingProxyType
 from typing import Any, Final
 from urllib.parse import urlparse
 
@@ -256,6 +257,25 @@ def normalize_llm_passthrough_logging_payload(
             )
             standard_logging_response_object = comprehend_medical_handler_result["result"]  # rebind-ok: elif-chain
             kwargs = comprehend_medical_handler_result["kwargs"]  # rebind-ok: elif-chain contract
+        elif self.is_typesafe_route(custom_llm_provider):
+            from .llm_provider_handlers.typesafe_passthrough_logging_handler import (
+                TypeSafePassthroughLoggingHandler,
+            )
+
+            typesafe_handler_result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
+                httpx_response=httpx_response,
+                response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}),
+                logging_obj=logging_obj,
+                url_route=url_route,
+                result=result,
+                start_time=start_time,
+                end_time=end_time,
+                cache_hit=cache_hit,
+                request_body=request_body,
+                **kwargs,
+            )
+            standard_logging_response_object = typesafe_handler_result["result"]
+            kwargs = typesafe_handler_result["kwargs"]
         elif self.is_vertex_ai_live_route(url_route):
             from .llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import (
                 VertexAILivePassthroughLoggingHandler,
@@ -389,6 +409,9 @@ def is_assemblyai_route(self, url_route: str):
     def is_comprehend_medical_route(self, custom_llm_provider: str | None) -> bool:
         return custom_llm_provider == "comprehendmedical"
 
+    def is_typesafe_route(self, custom_llm_provider: str | None) -> bool:
+        return custom_llm_provider == "typesafe"
+
     def is_langfuse_route(self, url_route: str):
         parsed_url: Final = urlparse(url_route)
         for route in self.TRACKED_LANGFUSE_ROUTES:

litellm/types/utils.py

@@ -348,6 +348,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
             "audio_transcription",
             "audio_speech",
             "responses",
+            "evaluation",
             "ocr",
             "realtime",
         ]

model_prices_and_context_window.json

@@ -69184,6 +69184,27 @@
         "supports_reasoning": true,
         "supports_vision": true
     },
+    "typesafe/jev-1.13.0": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-latest": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-preview": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
     "wandb/zai-org/GLM-5.3-Flash": {
         "cache_read_input_token_cost": 5e-08,
         "input_cost_per_token": 1.5e-07,

model_prices_and_context_window.schema.json

@@ -427,6 +427,7 @@
             "chat",
             "completion",
             "embedding",
+            "evaluation",
             "guardrail",
             "image_edit",
             "image_generation",