From 492891cad8915c2a19a675f3b603fbeed29c711b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 23 May 2026 12:15:41 -0700 Subject: [PATCH 001/109] CI: copy of #25177 (OCI GenAI: embeddings, streaming/reasoning fixes, model catalog) (#28223) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(opentelemetry): JSON-serialize dict metadata fields for OTEL span attributes (#27451) (#27455) Squash-merged by litellm-agent from Anai-Guo's PR. * feat(dashscope): add embeddings and reranks(qwen3-rerank) support via OpenAI-compatible endpoint (#27508) Squash-merged by litellm-agent from yimao's PR. * fix(vertex_ai/gemini): raise BadRequestError when image_url or url fi… (#24550) Squash-merged by litellm-agent from krisxia0506's PR. * fix(vertex_ai): raise error on mid-stream 429/error chunks instead of silently swallowing (#23711) Squash-merged by litellm-agent from krisxia0506's PR. * fix: raise BadRequestError for file content blocks missing 'file' sub… (#24503) Squash-merged by litellm-agent from krisxia0506's PR. * Fix Gemini MIME detection for extensionless GCS URIs (#27278) Squash-merged by litellm-agent from krisxia0506's PR. * fix(vertex_ai/partner_models): drop unused vertexai SDK gate from count_tokens (closes #28084) (#28107) Squash-merged by litellm-agent from voidborne-d's PR. * feat(chart): add support for autoscaling behavior in HPA (#27990) Squash-merged by litellm-agent from FabrizioCafolla's PR. * feat(proxy): add blocked flag to models for pause/resume from the UI (#27927) Squash-merged by litellm-agent from Cyberfilo's PR. * fix: pass socket timeouts to Redis cluster clients (#27920) Squash-merged by litellm-agent from tomdee's PR. * Fix/cache token (#28009) Squash-merged by litellm-agent from escon1004's PR. * fix(deepseek): forward reasoning_content in multi-turn thinking mode conversations (#28080) Squash-merged by litellm-agent from Divyansh8321's PR. * fix(guardrails): return HTTP 400 instead of 500 for blocked requests (#27617) * fix: reset org and tag budgets (#27326) * reset org budgets * reset tag budgets --------- Co-authored-by: Michael Riad Zaky * fix(ui): omit allowed_routes from key edit save when unchanged (#27553) * fix(ui): omit allowed_routes from key edit save when unchanged When a team admin opens Edit Settings on a key with key_type=AI APIs and saves without changing anything, the UI re-sends the existing allowed_routes value, which the backend's _check_allowed_routes_caller_permission gate rejects for non-proxy-admins (LIT-2681). Strip allowed_routes from the patch in handleSubmit when it deep-equals the original keyData.allowed_routes. The backend treats absence as "leave alone," so no-op saves now succeed for non-admins. Admins explicitly editing the field still send the new value. * fix(ui): order-insensitive allowed_routes diff + cover null-original case Address Greptile review: - Switch the "is allowed_routes unchanged" check to a Set-based comparison so a server-side reorder of the array doesn't register as a user edit and re-trigger LIT-2681. - Add two regression tests: (1) keyData.allowed_routes is null and the form is untouched — patch should strip the field; (2) server returned routes in a different order than the user originally entered — patch should still recognize the value as unchanged. * chore(ui): strip ticket refs and tighten comments in key edit fix - Remove internal-tracker references from in-code comments - Tighten the WHY comment in handleSubmit to two lines - Drop redundant test-block comments — test names already describe the case * fix(ui): annotate Set generic in allowed_routes diff to fix tsc * fix(guardrails): return HTTP 400 instead of 500 for guardrail-blocked requests GuardrailRaisedException and BlockedPiiEntityError both lacked a status_code attribute. When these exceptions reached the proxy exception handler (getattr(e, 'status_code', 500)), the fallback defaulted to HTTP 500 — making intentional guardrail blocks indistinguishable from server errors and causing unnecessary client retries. Changes: - Add status_code=400 (keyword-only) to GuardrailRaisedException - Add status_code=400 (keyword-only) to BlockedPiiEntityError - Update _is_guardrail_intervention() to recognize both exceptions so downstream loggers record 'guardrail_intervened' instead of 'guardrail_failed_to_respond' - Add 6 unit tests for default/custom status codes and getattr pattern - Strengthen existing blocked-action test with status_code assertion Fixes #24348 --------- Co-authored-by: Michael-RZ-Berri Co-authored-by: Michael Riad Zaky Co-authored-by: ryan-crabbe-berri Co-authored-by: Krrish Dholakia * fix(router/proxy): address Greptile P1+P2 review comments on PR #28161 - router: raise ServiceUnavailableError (503) instead of RouterRateLimitErrorBasic (429) when a specifically-addressed deployment is administratively blocked; 429 misleads retry-enabled clients into spinning forever against a paused model - proxy_server: compute get_fully_blocked_model_names() once before both branches in model_list() instead of duplicating the call in each branch - deepseek: upgrade silent debug log to warning when injecting placeholder reasoning_content so callers are clearly notified of degraded multi-turn quality - tests: update two blocked-deployment assertions to expect ServiceUnavailableError Co-authored-by: Cursor * fix: address bug detection findings (cache token order, mutable defaults) Co-authored-by: Yassin Kortam * fix: address bugs in async pass-through, anthropic cache token detection, rerank tests - async_get_available_deployment_for_pass_through: enforce blocked check on specific deployments - cost_calculator: detect anthropic-style usage by attribute presence (not truthiness) to avoid mixing OpenAI cached_tokens into anthropic normalization when read=0 - dashscope rerank tests: pass request to httpx.Response constructions for consistency Co-authored-by: Yassin Kortam * fix code qa * fix(vertex_ai/gemini): strip MIME parameters from GCS contentType GCS object metadata's contentType field can include parameters such as 'text/html; charset=utf-8'. Strip them in _apply_gemini_mime_type_aliases so downstream get_file_extension_from_mime_type sees a bare MIME type. Co-authored-by: Yassin Kortam * fix(vertex_ai/gemini): clarify mime-type error message string concatenation Co-authored-by: Yassin Kortam * feat(oci): add embeddings, fix streaming/reasoning, expand model catalog - Add OCIEmbedConfig with full Cohere embed support (7 models, batch up to 96) - Fix sync streaming: split SSE events on \n\n before JSON parsing - Fix reasoning models (Gemini 2.5, xAI Grok): make completionTokens and message optional in OCIResponseChoice to handle max_tokens exhausted on reasoning - Fix compartment_id resolution in chat transform to use resolve_oci_credentials - Fix tool call id: make OCIToolCall.id optional, generate UUID fallback for providers (Google via OCI) that omit it - Add OCI_KEY env var support for inline PEM keys - Fix datetime.utcnow() deprecation in request signing - Expand model catalog: 29 OCI models including Llama 4, Gemini 2.5, xAI Grok, Cohere Command A, and all Cohere embed variants - Add 37 live integration tests: sync/async completions for Meta/Google/xAI/Cohere, sync/async embeddings, tool use across all vendors, streaming, env var auth - Add 23 embed unit tests covering all transform and validation paths * fix(oci): remove dead OCI elif branch in utils.py, align async split_chunks with sync version * test(oci): add unit tests for split_chunks fix and no-duplicate-OCI-branch guard * fix(oci): address remaining bugs from issue #25082 — streaming signed body, Cohere stop sequences, hardcoded defaults - Bug 1: sync and async streaming paths now use signed_json_body when provided instead of re-serializing data with json.dumps() — the OCI RSA-SHA256 signature covers the exact request body bytes, so re-serializing produces an invalid sig - Bug 3: Cohere stop sequences now map to 'stopSequences' (was incorrectly 'stop') - Bug 4: removed hardcoded Cohere defaults (maxTokens=600, temperature=1, topK=0, topP=0.75, frequencyPenalty=0) that silently overrode user intent on every call - Added 6 unit tests covering all three fixes * fix(oci): comprehensive code quality pass — bugs, tests, schema accuracy - Fix Cohere tool call IDs (was always call_0; now UUID per call) - Fix TOOL_CALL finish reason mapping in both sync and streaming paths - Fix Cohere stop parameter mapping (stop → stopSequences) - Remove hardcoded Cohere defaults (maxTokens/topK/topP/frequencyPenalty) - Fix content[0] safety guard against empty content arrays - Fix streaming signed body used consistently (not re-serialized) - Raise OCIError (not bare Exception/ValueError) throughout - Centralize OCI_API_VERSION constant; import uuid at module level - Fix embed get_complete_url to strip trailing slashes from api_base - Fix OCIEmbedResponse schema: add inputTextTokenCounts (actual OCI field) - Fix embed usage computed from inputTextTokenCounts (sum of per-input counts) - Fix Cohere toolCallId included in tool result messages - Add OCIToolCall.id as Optional (absent in Google/xAI streaming chunks) - Update tests to reflect correct behavior (no hardcoded defaults, UUID ids, deferred credential validation, OCIError vs ValueError, real response schema) * test(oci): move integration tests to tests/llm_translation/ Addresses greptile P1: tests/test_litellm/ is for mock-only unit tests (make test-unit target). Real-network OCI tests now live in the correct location alongside other provider integration tests. * fix(oci): align types and transformation with official OCI SDK - Remove OCIVendors.GEMINI — apiFormat="GEMINI" is invalid; all non-Cohere models use apiFormat="GENERIC" - Add toolChoice, logitBias, logProbs to OCIChatRequestPayload so params present in the mapping are no longer silently dropped by Pydantic - Exclude n→numGenerations from Cohere param map (not a Cohere API field) - Fix CohereToolResult: change callId/result to call/outputs matching the OCI SDK's CohereToolResult structure - Fix CohereToolMessage: replace non-existent toolCallId with toolResults list; update adapt_messages_to_cohere_standard to build proper tool-result history entries by resolving tool call name+params from preceding assistant messages - Map generic-model stream finish reasons to OpenAI convention (COMPLETE→stop, MAX_TOKENS→length, TOOL_CALLS→tool_calls), consistent with the existing Cohere streaming path - Add optional id field to OCIEmbedResponse so valid API responses carrying an id are not rejected by the Pydantic model * fix(oci): use 'output' key in Cohere tool result outputs (matches reference impl) * fix(oci): port schema/type utilities from langchain-oracle reference impl - Add resolve_oci_schema_refs: inline $ref/$defs — OCI rejects JSON Schema refs - Add resolve_oci_schema_anyof: flatten Optional[T] anyOf (Pydantic v2 emits these) - Add sanitize_oci_schema: strip title, normalise null types, ensure array items - Add OCI_JSON_TO_PYTHON_TYPES: Cohere expects Python type names (str/int/float), not JSON Schema names (string/integer/number) - Add enrich_cohere_param_description: embed enum/format/range/pattern constraints into description since CohereParameterDefinition has no dedicated fields - Apply all of the above in adapt_tool_definitions_to_cohere_standard and adapt_tool_definition_to_oci_standard - Fix toolChoice conversion: map OpenAI string ('auto','none','required') to OCI dict form ({"type":"AUTO"} etc.) — the API rejects plain strings - Update unit test expectations to match correct Python type names and enriched descriptions * refactor(oci): split transformation.py into cohere.py and generic.py transformation.py was 1 243 lines doing too many jobs. Split along the same boundaries as the langchain-oracle reference (providers/cohere.py, providers/generic.py): chat/cohere.py — Cohere message/tool building, response + stream parsing chat/generic.py — Generic message/tool building, response + stream parsing transformation.py — thin OCIChatConfig orchestrator + OCIStreamWrapper Public symbols (OCIChatConfig, OCIStreamWrapper, adapt_messages_to_*, OCIRequestWrapper, version, …) remain importable from transformation.py for backward compatibility. OCIStreamWrapper gains delegating shims for _handle_cohere_stream_chunk and _handle_generic_stream_chunk so existing test call sites keep working unchanged. transformation.py: 1 243 → 620 lines * refactor(oci): principal-level code quality pass - Remove _extract_text_content duplication — single definition in cohere.py, imported where needed; instance method on OCIChatConfig eliminated - Move cryptography imports to module level with _CRYPTOGRAPHY_AVAILABLE flag and _require_cryptography() guard; no more re-import on every signing call - Move litellm version import to module level via litellm._version; remove inline import inside validate_oci_environment - sign_with_manual_credentials now returns Tuple[dict, bytes] matching sign_with_oci_signer — asymmetry eliminated, Optional[bytes] guards removed throughout stream wrappers (signed_json_body: bytes = b"") - Rename _openai_to_oci_cohere_param_map → openai_to_oci_cohere_param_map for consistency with openai_to_oci_generic_param_map - Remove double-key bug in map_openai_params where responseFormat was stored under both OCI and OpenAI key names simultaneously - Remove delegating shims (adapt_messages_to_cohere_standard, adapt_tool_definitions_to_cohere_standard, _handle_generic_stream_chunk) from OCIChatConfig/OCIStreamWrapper; tests now import directly from cohere.py and generic.py where symbols live - Trim __all__ to 7 genuine public symbols; remove the 13-symbol list that existed only to support test imports - Collapse per-model integration test classes into pytest.mark.parametrize; CHAT_MODELS list is the single source of truth for model-specific config - Black + Ruff clean across all OCI files * fix(oci): address PR review findings - types/llms/oci.py: add "TOOL_CALL" to CohereChatResponse.finishReason Literal so Pydantic does not raise ValidationError on non-streaming Cohere tool-use calls (Greptile P1) - test_oci_cohere_tool_calls.py: add test covering TOOL_CALL finish reason - model_prices_and_context_window.json: remove 6 duplicate oci/cohere.embed-* keys that were silently overridden by the more complete entries already present in the file (Greptile P1) - common_utils.py: move OCI_API_VERSION here from chat/transformation.py so embed/transformation.py does not need to import chat/transformation; change Protocol stub body from ... to pass (CodeQL "statement no effect"); add comment to sha256_base64 clarifying it implements OCI HTTP signing spec, not password hashing (CodeQL false positive) - chat/transformation.py: import CustomStreamWrapper from litellm_core_utils.streaming_handler instead of litellm.utils to reduce import cycle depth (CodeQL cyclic import) - chat/cohere.py, chat/generic.py: import Usage and ChatCompletionMessageToolCall from litellm.types.utils instead of litellm.utils for the same reason - embed/transformation.py: import OCI_API_VERSION from common_utils instead of chat/transformation (removes the embed→chat import edge) * test(oci): add unit tests to improve patch coverage - test_oci_common_utils.py (new): covers sha256_base64, build_signature_string, OCIRequestWrapper.path_url, resolve_oci_credentials, get_oci_base_url, validate_oci_environment, sign_with_oci_signer error paths, sign_oci_request routing, load_private_key_from_file error paths, resolve_oci_schema_refs (including circular ref and external $ref), resolve_oci_schema_anyof, sanitize_oci_schema (all branches), enrich_cohere_param_description - test_oci_generic_chat.py (new): covers content-message error paths (non-dict item, unsupported type, non-string text, invalid image_url), tool-call validation error paths, adapt_messages_to_generic_oci_standard error paths, handle_generic_response (None message, text content, tool calls), handle_generic_stream_chunk (finish reasons, streaming tool calls), OCIStreamWrapper non-string chunk error - test_oci_chat_transformation.py: add error paths for validate_environment (empty messages), transform_request (missing compartment_id, Cohere without user messages), transform_response (error key), map_openai_params (unsupported param with and without drop_params), tool_choice string mapping - test_oci_cohere_tool_calls.py: add edge cases for stream chunk finish reasons (TOOL_CALL, MAX_TOKENS, unknown), _extract_text_content with non-dict list items and non-string input, adapt_messages_to_cohere_standard with malformed JSON tool arguments * fix(oci): rename supports_streaming to supports_native_streaming in model prices The JSON schema for model_prices_and_context_window.json uses `supports_native_streaming` (not `supports_streaming`) and has `additionalProperties: false`. Rename the field across all OCI entries to pass the schema validation test. * test(oci): add 67 tests targeting uncovered happy paths for coverage Boost patch coverage on the four lowest-coverage OCI files: - common_utils.py: sign_with_manual_credentials (oci_key / oci_key_file paths), sign_oci_request routing, _require_cryptography - generic.py: adapt_messages_to_generic_oci_standard (all roles), adapt_tool_definition_to_oci_standard, adapt_tools_to_openai_standard, handle_generic_stream_chunk text/finish-reason paths - cohere.py: _extract_text_content, adapt_messages_to_cohere_standard (all roles including tool results), handle_cohere_response / handle_cohere_stream_chunk all finish-reason branches - transformation.py: get_vendor_from_model, OCIChatConfig._get_optional_params (toolChoice string→dict, responseFormat, tools for both vendors), transform_request for GENERIC model, get_sync/async_custom_stream_wrapper with mocked HTTP, OCIStreamWrapper.chunk_creator happy paths * fix(oci): suppress CodeQL false positive on sha256_base64 (OCI HTTP signing, not password hashing) * fix(oci): remove 6 duplicate model price entries and reconcile conflicting values Six OCI chat model keys appeared twice in model_prices_and_context_window.json with conflicting pricing/context data (JSON parsers silently discard the first). Remove the first-occurrence entries and update the surviving entries: - meta.llama-4-maverick / llama-4-scout: keep updated entries (free preview pricing, larger context windows, vision support) - meta.llama-3.1-70b: keep original pricing, restore supports_native_streaming - google.gemini-2.5-{flash,pro,flash-lite}: keep OCI pricing page values, restore supports_native_streaming * fix(oci): route GPT-5 family to maxCompletionTokens GPT-5 / GPT-5-mini / GPT-5-nano / GPT-5.5 on OCI reject "maxTokens" with HTTP 400: Invalid 'maxTokens': Unsupported parameter: 'maxTokens' is not supported with this model. Use 'maxCompletionTokens' instead. (Same convention as OpenAI's reasoning-API contract.) Add a model-aware rename in OCIChatConfig._get_optional_params so the request payload uses maxCompletionTokens when the model id starts with openai.gpt-5. Regular Llama / Cohere / Gemini / GPT-4.x continue to use maxTokens unchanged. Also widen OCIChatRequestPayload to carry the new optional field so it survives Pydantic serialization. Verified live against OCI us-chicago-1: - openai.gpt-5, gpt-5-mini, gpt-5-nano, gpt-5.5 all return 200 - Full feature sweep on gpt-5.5 (basic, system, multi-turn, streaming, tools, usage) all green - meta.llama-3.3-70b-instruct still uses maxTokens (no regression) 4 new unit tests cover the helper, the routing in both pre- and post-translation states, and Pydantic serialization. * ci(oci): fix CI failures — black formatting + recursive_detector ignore - Run black on litellm/llms/oci/common_utils.py + 3 OCI test files that drifted out of black-compliance during the rebase. - Add the three bounded recursive functions in oci/common_utils.py (`_resolve`, `resolve_oci_schema_anyof`, `sanitize_oci_schema`) to the recursive_detector IGNORE_FUNCTIONS list. All three are bounded: `_resolve` uses a `resolving_stack` cycle guard; the other two are bounded by JSON-schema tree depth (no cycles in well-formed input), matching the pattern of the existing OCI/Vertex schema walkers already on the list. * fix(oci): silence MyPy errors in cohere.py — typed-dict access Two errors flagged by `lint` CI: llms/oci/chat/cohere.py:73: "object" has no attribute "__iter__" llms/oci/chat/cohere.py:119: No overload variant of "get" of "dict" matches argument types "object", "CohereToolCall" Both stem from `msg.get("tool_calls")` / `msg.get("tool_call_id")` returning `object` per the AllMessageValues TypedDict union. Bind to `Any` locally for the iteration and coerce the lookup key with `str()`, removing the now-unused `# type: ignore` on those lines. No behaviour change — pure type-narrowing for the type checker. * fix(oci): silence CodeQL py/weak-sensitive-data-hashing on sha256_base64 CodeQL's taint analysis traces request bodies back to environment-loaded secrets and flags `hashlib.sha256(body).digest()` as `py/weak-sensitive-data-hashing` — even though SHA-256 is the algorithm mandated by the OCI HTTP request signing spec for the `x-content-sha256` header (not a password/secret hash). The previous suppression used legacy `# lgtm[...]` syntax which the modern CodeQL action ignores. Switch to Python's standard `hashlib.sha256(..., usedforsecurity=False)` (Python 3.9+) which CodeQL honours as a non-security declaration. Behaviour unchanged. * feat(oci): add reasoning_effort passthrough — only true missing primitive OCI's GenericChatRequest exposes a reasoningEffort field (NONE/MINIMAL/LOW/MEDIUM/HIGH) that's the single biggest cost knob for reasoning-capable models on the service: - GPT-5 family - Gemini 2.5 - Grok reasoning variants (3-mini, 4-fast, 4.20) - Cohere Command-A-Reasoning Setting reasoning_effort=LOW typically cuts reasoning-token spend 5-10× vs the default. Without exposing this, litellm users had no way to tune cost-vs-quality on these models. The other GenericChatRequest fields (verbosity, parallel_tool_calls, logit_bias, n, metadata, web_search_options, prediction) are not exposed because they are not missing primitives — they either duplicate prompt-engineering, framework-level controls, or are too niche to justify the maintenance surface. We only ship what users genuinely can't accomplish another way. Excluded from the Cohere v1 param map: CohereChatRequest has no reasoningEffort field, and Cohere reasoning models (cohere.command-a-reasoning) use COHEREV2 which is a separate request type not covered by this PR. Verified live: GPT-5.5 + reasoning_effort="HIGH" sends {"reasoningEffort": "HIGH"} on the wire and OCI accepts the request. * feat(oci): reasoning_effort + reasoning_tokens for OCI GenAI Three small additions for OCI reasoning models, requested by users testing the PR in production fork builds: 1. **reasoning_effort param mapping (GENERIC vendors).** OCI expects uppercase levels ("LOW"/"MEDIUM"/"HIGH"/"NONE") on `reasoningEffort`, but OpenAI-compatible clients send lowercase. Mapped + uppercased in `_get_optional_params`. Marked unsupported on Cohere V1/V2 since OCI Cohere has no reasoning models (avoids Pydantic validation failure on CohereChatRequest). 2. **"disable" → "NONE" mapping.** OpenAI uses "disable" to turn off reasoning; OCI uses "NONE". Without this, callers get a 400. 3. **reasoning_tokens propagated to Usage.** OCI returns `completionTokensDetails.reasoningTokens` but it wasn't being passed to LiteLLM's Usage object. Now flows through to `Usage.completion_tokens_details.reasoning_tokens` so callers can track reasoning token consumption for cost/observability. Tests: 7 new unit tests in TestOCIReasoningEffort covering upper/lower case, "disable"→"NONE", Cohere drop/raise paths, and reasoning_tokens extraction (with and without completionTokensDetails). 5 new live integration tests against xai.grok-3-mini in us-chicago-1 verifying the full request/response loop end-to-end. Existing test_transform_response_simple_text assertion that completion_tokens_details was None has been updated to assert reasoning_tokens flows through. Verified live on xai.grok-3-mini: reasoning_effort=low → OCI accepts "LOW", returns reasoningTokens=316 in usage. reasoning_effort=disable → OCI accepts "NONE". Full suite: 370/370 unit + 51/51 integration. * fix(codeql): re-scope py/weak-sensitive-data-hashing exclusion to OCI signing file CodeQL's taint analysis re-fires the `py/weak-sensitive-data-hashing` alert at `litellm/llms/oci/common_utils.py:103` whenever upstream code paths into the OCI signing module change (touching `transformation.py` opens new flow paths that CodeQL re-evaluates from scratch). The `hashlib.sha256(..., usedforsecurity=False)` declaration silences the direct-call form of the query but not the taint-flow form. SHA-256 here is mandated by the OCI HTTP signing specification for the x-content-sha256 content-integrity header — not for password storage: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/signingrequests.htm CodeQL has no per-query path filter and GitHub Code Scanning ignores inline lgtm/codeql comments, so path-ignoring this single ~560-line signing utility file is the narrowest available suppression. All other files retain full coverage of py/weak-sensitive-data-hashing — including litellm/proxy/utils.py where the rule legitimately applies. This restores the NEUTRAL CodeQL state the PR had on prior commits (see `2111c98af7` for the same approach on the previous branch evolution that the cherry-pick was rebased onto a different baseline). * fix(oci): drop duplicate text on Cohere streaming terminal chunk OCI Cohere's terminal SSE event re-sends the full assembled response in `text` alongside a populated `chatHistory`. Emitting that text as another delta concatenates the entire response onto the already-streamed output (e.g. "How can I help?How can I help?"). Use `chatHistory is not None` as the discriminator for the consolidated terminal event — `finishReason` is a weaker signal that could in principle appear on a non-consolidated chunk. The two coincide today; this preserves correctness if OCI ever ships finishReason on an incremental chunk. Adds a live-OCI integration regression test that compares streamed vs non-streamed length and asserts the response prefix appears only once. Verified to fail under the previous code with the exact reported reproduction: 'Hello! How can I help you today?Hello! How can I help you today?'. Reported by @gotsysdba on PR #25177. * fix(oci): buffer SSE stream across HTTP read boundaries The old split_chunks helper split each individual HTTP read on "\n\n", which assumed SSE event boundaries always aligned with read boundaries. In practice the OCI streaming endpoint delivers events that may: - straddle two reads (chunk_creator gets a truncated JSON and crashes) - arrive separated by a single "\n" instead of "\n\n" - share a read with multiple complete events Replace the inline split with module-level helpers _iter_sse_events (sync) / _aiter_sse_events (async) that maintain a buffer across reads, split on any newline, and yield only complete "data:" lines. Add 25 regression tests covering event-split-across-reads, tiny-chunk reads, single-newline separators, keepalive/comment lines, trailing partial events flushed at EOF, "\r\n" line endings, and an end-to-end smoke test that feeds an awkwardly-chopped payload through the splitter into OCIStreamWrapper.chunk_creator. Reported by John Lathouwers. * test(oci): repoint TestOCIKeyNormalization to sign_with_manual_credentials The signing helper moved from OCIChatConfig._sign_with_manual_credentials to a module-level sign_with_manual_credentials in common_utils.py. Four tests in TestOCIKeyNormalization still called the old method: - 2 failed outright with AttributeError - 2 passed by accident because they used pytest.raises(Exception), which happily caught the AttributeError instead of exercising the intended OCIError path Repoint all four to the new module-level function so they exercise the actual oci_key type-validation branch. * fix(oci): validate oci_region before URL interpolation to prevent SSRF Anchor oci_region to ^[a-z][a-z0-9-]{0,30}[a-z0-9]$ inside get_oci_base_url so user-supplied regions that would redirect the signed request to an attacker-controlled host (e.g. 'evil.com/#') fail with HTTP 400 before the URL or signature is built. Empty string still falls back to the us-ashburn-1 default, so existing callers are unaffected. * test(audio): skip when gpt-4o-audio-preview is unavailable upstream OpenAI retired `gpt-4o-audio-preview` (404 model_not_found in CI as of 2026-05-19), and the existing try/except in these tests only re-raised on 'openai-internal' errors. Other exceptions were silently swallowed, so the next line ran with an unbound `response`/`completion` and failed with an unrelated UnboundLocalError that masked the real cause. Extend the skip condition to also cover model_not_found / 'does not exist' so the suite reports the upstream outage cleanly, matching the pattern used in ce87c41 for the realtime and nvidia_nim rerank tests. Re-raise unknown exceptions instead of falling through. * fix(oci/router): catalog-driven maxCompletionTokens; generic blocked-deployment message - Drive OCI maxCompletionTokens via supports_reasoning from the model catalog instead of a hardcoded openai.gpt-5 prefix. Add OCI GPT-5 family entries (gpt-5, gpt-5-mini, gpt-5-nano) with supports_reasoning: true. Gate the override to non-Cohere vendor so Cohere reasoning models keep maxTokens (Cohere endpoint does not accept maxCompletionTokens). - Replace proxy-specific 'Contact your proxy admin' phrasing in the four Router blocked-deployment ServiceUnavailableError messages with neutral SDK-appropriate text. * fix(oci/cohere): guard handle_cohere_response against missing usage * fix(oci): address bug review findings in chat transformation - Cohere param map: keep tool_choice/n as False (not omitted) so unsupported params are dropped or rejected rather than silently passed through. - get_complete_url: when an explicit api_base/litellm.api_base is provided, use it as-is instead of unconditionally appending /20231130/actions/chat (mirrors the embed config behavior). - Cohere stream: require both chatHistory and finishReason to be present to identify a terminal consolidation chunk, avoiding silent text suppression if chatHistory ever appears on a non-terminal chunk. - Generic usage: use 'is not None' for reasoningTokens so a legitimate value of 0 is preserved instead of being treated as absent. Co-authored-by: Yassin Kortam * fix(oci/cohere): emit tool calls in streaming and null content when text empty handle_cohere_response now sets message.content to None when the Cohere response text is empty, matching the OpenAI convention for tool-call-only responses. handle_cohere_stream_chunk now extracts toolCalls — both directly from the chunk and from the terminal chunk's chatHistory CHATBOT message — and emits them in the delta. Previously, CohereStreamChunk lacked a toolCalls field, so any tool calls in the stream were silently dropped. * fix(oci): preserve tool results, embed URL path, and generic finish reason - Use SerializeAsAny on CohereChatRequest.chatHistory so subclass-specific fields like CohereToolMessage.toolResults are not dropped during Pydantic v2 serialization. - Make OCIEmbedConfig.get_complete_url append the /20231130/actions/embedText action path consistently with chat, so setting litellm.api_base to the region inference base URL no longer posts to the bare hostname. - Map OCI finishReason (COMPLETE / MAX_TOKENS / TOOL_CALLS) to OpenAI finish_reason values in handle_generic_response, mirroring the streaming handler and the Cohere non-streaming handler. Co-authored-by: Yassin Kortam * fix(oci/generic): silence mypy assignment error on dynamic finish_reason * fix(oci/embed): always set usage on embedding response Co-authored-by: Yassin Kortam * fix(oci/chat): append /20231130/actions/chat to explicit api_base Restore the embed-style behavior so OCIChatConfig.get_complete_url always appends the OCI GenAI chat path. Routing through get_oci_base_url ensures the optional explicit api_base has its trailing slash stripped before the suffix is joined, matching the embed config and the test_respects_explicit_api_base expectation. * fix(oci/cohere): mark logprobs/logit_bias unsupported and normalize unknown stream finish reasons Co-authored-by: Yassin Kortam * fix(oci/cohere): preserve trailing tool result in chatHistory When the last message in the OpenAI-format input is a tool result (the standard agentic continuation pattern), the prior messages[:-1] slice silently dropped that tool result from chatHistory and the model never saw it. Excluding the last user message by index instead keeps tool results that trail the last user turn intact. * fix(main): remove dead OCI embedding elif block The earlier elif at line 5119 already routes OCI embeddings through the base HTTP handler with the headers None-guard, so the later identical block was unreachable dead code. * test(oci): move integration tests out of llm_translation mock-only folder Greptile flags tests/llm_translation/ as mock-only via a project-specific rule; relocate the live-network OCI integration suite to tests/integration/ and adjust the in-file sys.path / run instructions accordingly. * fix(oci/cohere): suppress tool calls on stream terminal consolidation chunk The terminal SSE event re-sends the full assembled response in both `text` and `chatHistory`. The existing logic already suppresses `text` to avoid double-emit, but tool calls extracted from the terminal chunk (via `typed_chunk.toolCalls` or the `chatHistory` CHATBOT fallback) would still be re-emitted with fresh uuid4 IDs. If OCI Cohere ever streams tool calls progressively in intermediate chunks (now possible since CohereStreamChunk has a toolCalls field), this would cause downstream agentic frameworks to execute each tool call twice. Suppress tool calls on the terminal consolidation chunk for the same reason `text` is suppressed. Co-authored-by: Yassin Kortam * fix(oci,httpx): normalize finish_reason, preserve response_format, fix sync embed JSON content-type - cohere.py / generic.py: normalize unknown OCI finishReason values (ERROR, ERROR_TOXIC, CONTENT_FILTERED, USER_CANCEL, ...) to 'stop' in non-streaming and streaming generic handlers, matching the streaming Cohere handler so downstream consumers switching on finish_reason aren't broken by raw OCI values. - transformation.py: restore the dual-key alias so optional_params still carries the original 'response_format' key alongside the OCI-mapped 'responseFormat'. Downstream litellm framework code (json_mode detection, logging) inspects 'response_format' after map_openai_params runs. - llm_http_handler.py: make the sync embedding path mirror the async path — when sign_request returns no signed_body, send via json=data (which sets Content-Type: application/json) instead of data=json.dumps(data) which doesn't. Removes a sync/async behavioural asymmetry for non-OCI providers that adopt the sign_request pattern. Co-authored-by: Yassin Kortam * fix(oci): clean up OCIChatConfig init, normalize generic stream finish reasons, correct embed sign_request return type - Replace fragile setattr(self.__class__, ...) pattern in OCIChatConfig.__init__ with a @property for has_custom_stream_wrapper, matching the pattern used by other providers. - Normalize unknown OCI finish reasons (e.g. ERROR, ERROR_TOXIC, USER_CANCEL) to 'stop' in handle_generic_stream_chunk, matching the existing Cohere stream handler behaviour. - Tighten OCIEmbedConfig.sign_request return type from Tuple[dict, Optional[bytes]] to Tuple[dict, bytes] — sign_oci_request never returns None for the body, and this matches OCIChatConfig.sign_request. Co-authored-by: Yassin Kortam * fix(oci): strip trailing action path in get_oci_base_url to avoid URL doubling A fully-formed OCI endpoint URL (e.g. https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/chat) passed via api_base previously had the action path appended a second time by get_complete_url in both chat and embed configs, yielding a 404. get_oci_base_url now strips a trailing /20231130/actions/ so callers can always append the action path safely. * fix(httpx): preserve sync embed data= kwarg to avoid breaking mock-based tests The earlier sync_httpx_client.post() call passed data=json.dumps(data), which downstream embedding tests assert on (e.g. tests for hosted_vllm, jina_ai, watsonx). Switching to json=data changed the kwarg name and broke those tests. The OCI signed_body path keeps using data=signed_body and is unaffected. * fix(oci): stable tool-call ids across stream chunks; lenient Cohere finishReason - Replace random uuid4 per chunk with a deterministic content-derived digest for synthetic tool-call ids in both Cohere and Generic OCI handlers. Previously, when OCI omitted 'id' (always for Cohere, often for Generic streaming deltas), every chunk for the same logical tool call received a new uuid, causing downstream stream-mergers (which key off id) to treat each fragment as a distinct call. - Relax CohereChatResponse.finishReason from a strict Literal[...] to Optional[str], matching CohereStreamChunk.finishReason. The handle_cohere_response 'elif oci_finish_reason is not None' fallback was previously unreachable because Pydantic raised ValidationError on any unknown value before the fallback executed. Now non-streaming responses degrade unknown reasons to 'stop' just like the streaming path. Co-authored-by: Yassin Kortam * fix(oci/embed): validate OCI credentials in validate_environment Mirror OCIChatConfig.validate_environment so embedding requests fail fast with a clear error when oci_user/oci_fingerprint/oci_tenancy/ oci_compartment_id or an oci_key/oci_key_file is missing, instead of deferring the failure until sign_request. Co-authored-by: Yassin Kortam * test(oci/embed): expect OCIError from validate_environment when credentials are missing OCIEmbedConfig.validate_environment now raises eagerly (mirroring OCIChatConfig) when oci_user/oci_fingerprint/oci_tenancy/oci_compartment_id or oci_key/oci_key_file is missing. Update the test to match. * fix(oci): polish stream chunk handling and signed body default - cohere stream terminal consolidation now emits content=None instead of "" - drop redundant index truthiness check (None is already replaced with 0) - accept both "TOOL_CALL" and "TOOL_CALLS" finish reasons in cohere - signed_json_body defaults to None and uses explicit None check, so an explicitly empty bytes body wouldn't be silently re-serialized Co-authored-by: Yassin Kortam * fix(oci/chat): catch pydantic ValidationError when parsing OCI responses Pydantic v2 raises ValidationError (not TypeError) when field validation fails, so malformed OCI completion responses or stream chunks would propagate unhandled out of handle_generic_response, handle_generic_stream_chunk, and handle_cohere_stream_chunk. Widen the except clauses to also catch ValidationError so callers get a clean OCIError. * fix(oci/catalog): real prices for Llama 4, drop zero-cost OCI OpenAI entries Zero-cost catalog entries (input_cost_per_token=0, output_cost_per_token=0) make proxy spend tracking silently report $0 for these paid OCI models, so any caller can drive them without decrementing a budget. For Llama 4 Maverick and Scout, OCI charges the same character-based rate as Llama 3.3 70B ($0.0018 per 10,000 characters), so use the same per-token price as the existing oci/meta.llama-3.3-70b-instruct entry (7.2e-07 in/out). For oci/openai.gpt-5, gpt-5-mini, gpt-5-nano, gpt-oss-120b, and gpt-oss-20b, no public per-token pricing is available; drop the entries so operators must register them with explicit custom pricing. The existing GPT-5 reasoning test fixture already injects synthetic entries when the catalog omits them, so the chat transformation's supports_reasoning lookup keeps working in tests. * fix(oci/chat): wrap CohereChatResult construction in try/except Match the handle_generic_response pattern: surface OCIError with the upstream status code instead of letting a raw pydantic.ValidationError propagate when the Cohere response payload is malformed. * fix(oci): harden Cohere stream/finish-reason and dedupe maxTokens param mapping - Cohere stream: track per-stream tool-call emission and only suppress the terminal consolidation chunk's tool calls once they've been seen earlier. Prevents silent drop if tool calls are delivered exclusively on the terminal chunk. - Cohere stream: emit content=None (not "") on non-terminal text-free chunks (e.g. tool-call-only / keep-alive) so downstream consumers that distinguish missing vs explicitly-empty deltas behave correctly. - Generic handlers: accept singular TOOL_CALL finish reason in addition to TOOL_CALLS, matching the Cohere handlers. - _get_optional_params: when both max_tokens and max_completion_tokens are provided, explicitly prefer max_completion_tokens instead of relying on dict iteration order. Co-authored-by: Yassin Kortam * fix(oci): emit content=None instead of empty string for text-free generic stream chunks Co-authored-by: Yassin Kortam * test(oci): expect content=None for text-free generic stream chunks handle_generic_stream_chunk now emits content=None instead of empty string when a chunk carries no text parts. Update the corresponding no-message test to match. * codeql: narrow OCI sha256 suppression to query-filter, not whole file paths-ignore was suppressing every CodeQL query on litellm/llms/oci/common_utils.py, hiding all future findings in a security-critical file (private key loading, credential resolution, URL construction, RSA signing). Move the suppression for py/weak-sensitive-data-hashing into query-filters so common_utils.py remains fully analyzed by every other query. Co-authored-by: Yassin Kortam * fix(oci): use locale-independent RFC 7231 date for manual signing email.utils.formatdate(usegmt=True) emits canonical English weekday/ month abbreviations regardless of system locale, so signature verification doesn't break on non-en_US deployments. * fix(oci): strip 'oci/' prefix in get_vendor_from_model Previously, get_vendor_from_model split on '.' without stripping the optional 'oci/' provider prefix, so 'oci/cohere.command-a-03-2025' was routed through the GENERIC pipeline instead of COHERE. Co-authored-by: Yassin Kortam * codeql: scope OCI sha256 suppression to common_utils.py via filter-sarif Replace the global query-filters exclude for py/weak-sensitive-data-hashing with a SARIF post-filter that only drops the alert when it originates from litellm/llms/oci/common_utils.py, keeping the rule active on every other SHA-256 callsite in the repository. * Fix OCI chat bugs: tool_calls None key, dead max_tokens dedup, single-event stream text suppression - handle_cohere_response: omit tool_calls key from message dict when None, matching the generic handler's behaviour and avoiding tripping consumers that key off 'tool_calls' in message. - _get_optional_params: remove dead prefer_max_completion branch. By the time this helper runs, map_openai_params has already collapsed max_tokens/max_completion_tokens onto the OCI alias, so the OpenAI-key membership check is unreachable. - handle_cohere_stream_chunk: add prior_text_emitted parameter mirroring prior_tool_calls_emitted. The terminal consolidation chunk's text is only suppressed when prior deltas already emitted text — otherwise (degenerate single-event stream) the text passes through so the response content isn't silently lost. OCIStreamWrapper now tracks emitted text alongside emitted tool calls. Co-authored-by: Yassin Kortam * fix(oci): preserve all text parts in generic response and emit SYSTEM role for Cohere - handle_generic_response: iterate all content parts and concatenate text (matches the streaming handler) so non-leading text parts are not lost and a leading non-text part does not suppress trailing text. - adapt_messages_to_cohere_standard: emit CohereSystemMessage for system messages so direct callers do not silently drop them. The Cohere request builder filters system messages before calling this helper to avoid duplicating preambleOverride content into chatHistory. Co-authored-by: Yassin Kortam * fix(oci): normalise dict-format tool_choice to OCI flat uppercase shape The OCI Generative AI API only accepts toolChoice values of the form {"type": "AUTO"|"NONE"|"REQUIRED"} or {"type": "FUNCTION", "name": ""}. The previous conversion only handled string tool_choice values, so OpenAI's standard dict shape {"type": "function", "function": {"name": ""}} passed through unchanged and was rejected by OCI with a 400. Normalise the dict shape by uppercasing the discriminator and hoisting the function name to the top level. Also accept dict variants of the non-function selectors (e.g. {"type": "auto"}). * test(oci): exercise system-message filtering at transform_request boundary adapt_messages_to_cohere_standard now emits SYSTEM-role entries by design so direct callers don't silently drop system content. The Cohere request builder filters system messages before calling the helper and routes them into preambleOverride, so the user-visible 'no SYSTEM in chatHistory' guarantee holds at the transform_request boundary, where the test should live. * fix(oci/chat): extract tool_choice/response_format helpers to satisfy PLR0915 _get_optional_params exceeded ruff's 50-statement cap. The toolChoice and responseFormat normalisation blocks are self-contained mutations, so move them to module-level helpers. * fix(oci): normalize None finishReason in generic non-streaming handler; drop dead Cohere system-role branch Co-authored-by: Yassin Kortam * fix(oci/generic): silence mypy assignment error on cleared finish_reason * fix(docker): install libatomic in builder for prisma nodeenv binary The prebuilt node binary that prisma-python's nodeenv downloads links against libatomic.so.1, which Wolfi does not pull in via gcc/nodejs. Without this, fresh Docker builds (no GHA cache hit) fail at `prisma generate` with: node: error while loading shared libraries: libatomic.so.1 * fix(oci): raise on invalid tool_choice instead of silently passing OpenAI shape _normalize_tool_choice previously left an OpenAI-format dict in selected_params['toolChoice'] when the type was unrecognized or when 'FUNCTION' was given with a missing/empty name. OCI would then reject the request with a non-obvious error. Raise ValueError with a clear message in these cases. Co-authored-by: Yassin Kortam * fix(oci): raise OCIError instead of ValueError in _normalize_tool_choice Co-authored-by: Yassin Kortam * fix(oci/generic): declare non-security intent on sha256 for synthetic tool-call id * fix(oci): simplify _get_optional_params and reject invalid tool_choice types - Collapse the two-loop _get_optional_params into a single pass with clear precedence (OpenAI key wins over OCI alias; first OpenAI key reaching a given OCI target wins). Removes the redundant maxTokens special-case in the second loop and makes the map_openai_params / transform_request handoff easier to reason about. - Raise OCIError when _normalize_tool_choice sees an unexpected type (list, bool, int, ...) instead of silently letting it through to the OCI API where it would produce an opaque server-side error. Co-authored-by: Yassin Kortam * Remove no-op data['stream'] deletion in OCI stream wrappers Co-authored-by: Yassin Kortam * fix(oci): always send Cohere isStream field explicitly Match OCIChatRequestPayload by defaulting CohereChatRequest.isStream to False instead of None so model_dump(exclude_none=True) does not silently omit the field on non-streaming requests. Co-authored-by: Yassin Kortam * fix(oci): revert Cohere isStream to Optional[bool]=None to preserve omission semantics Co-authored-by: Yassin Kortam * fix(oci/generic): raise OCIError on empty choices instead of IndexError Pydantic accepts an empty choices list when validating OCICompletionResponse, so accessing chatResponse.choices[0] could raise an unhandled IndexError. Surface it as OCIError so the response error path is consistent with the existing (TypeError, ValidationError) guard. * fix(oci/cohere): map top_k -> topK so Cohere topK param is settable The Cohere param map (derived from the GENERIC map) had no entry for topK. Since the simplified _get_optional_params only iterates over param_map entries, callers had no way to pass topK to CohereChatRequest (neither via an OpenAI-style key nor via the OCI alias). Add 'top_k': 'topK' to the Cohere map only — OCIChatRequestPayload (GENERIC) has no topK field. _get_optional_params accepts both the OpenAI key (top_k) and the OCI alias (topK) in optional_params, so this covers both calling conventions. Co-authored-by: Yassin Kortam * fix(oci): tighten cohere stream dedup flags and forward stream args in embed signing Co-authored-by: Yassin Kortam * fix(oci/chat): reorder dict guard and wrap stream chunk json.loads - Move isinstance(response_json, dict) check before .get("error") so the guard runs before the attribute access it is supposed to protect. - Wrap json.loads in OCIStreamWrapper.chunk_creator with try/except so malformed SSE payloads surface as OCIError instead of a raw JSONDecodeError propagating out of the stream loop. * fix(oci/cohere stream): only flag text emitted on non-empty content An intermediate Cohere SSE chunk carrying text="" was flipping _cohere_text_emitted via the "is not None" check, which then caused the terminal consolidation chunk to drop its real text as a duplicate. Use a truthy check so only actual content marks the stream as having emitted text. * test(oci): end-to-end proxy integration test against real OCI GenAI Spins up the litellm proxy via the console-script entrypoint with a minimal OCI-only config and drives real OpenAI-shaped HTTP requests through it against OCI GenAI. Covers non-streaming chat, streaming chat, embeddings, and /v1/models for Cohere, Llama, Gemini, and Grok. Skips automatically when ~/.oci/config is absent or when the active profile uses session-token auth (the OCI provider currently only consumes OCI_* env vars; session tokens would need an in-process signer). API-key profiles work out of the box. * test(oci): move proxy integration test to tests/integration/ tests/llm_translation/ is mock-only; the OCI proxy integration test spawns a real proxy subprocess and makes live HTTP calls, so move it (and the companion config) to tests/integration/ alongside the existing test_oci_integration.py. * fix(oci): dedupe finish-reason mapping and batch Cohere tool results - Extract _normalize_oci_finish_reason helper so the four chat handlers (Cohere/GENERIC, sync/stream) share one OCI->OpenAI mapping instead of four near-identical if/elif chains. - Merge consecutive OpenAI tool-role messages into a single CohereToolMessage with multiple toolResults entries, matching the OCI Cohere API's expectation for parallel tool calls in one assistant turn. Co-authored-by: Yassin Kortam * fix(oci): drop dead Cohere toolChoice field and emit GENERIC tool-call dicts inline - Remove the unreachable toolChoice field from CohereChatRequest. The Cohere param map explicitly marks tool_choice as unsupported, so the field can never be populated through the normal optional_params flow and only confused the public model surface. - Build GENERIC stream tool-call dicts inline (id/type/function shape) instead of round-tripping through ChatCompletionMessageToolCall and model_dump(). Matches handle_cohere_stream_chunk so downstream stream-mergers see the same minimal payload regardless of which vendor produced the chunk. Co-authored-by: Yassin Kortam * fix(docker): drop redundant libatomic from non_root builder litellm_internal_staging already fixes the prisma `nodeenv` build failure at the root cause by restoring `npm` to the builder (#28519): with npm on PATH, prisma-python uses the system Node and never downloads the nodeenv binary that links against libatomic.so.1. After merging internal_staging the libatomic line is dead weight, so remove it. https://claude.ai/code/session_01SwKzxRxgUhLFyyEf4UV812 * fix(oci/catalog): add openai.gpt-5{,-mini,-nano} entries with supports_reasoning Without these catalog entries, supports_reasoning(model='openai.gpt-5*', custom_llm_provider='oci') returned False, so _model_uses_max_completion_tokens fell back to the default and OCI rejected the request with HTTP 400 ('Use maxCompletionTokens instead.'). Add the three entries so the catalog-driven maxCompletionTokens routing works against a stock LiteLLM install. Also reword the test fixture docstring — the bundled backup now actually ships these entries, so the fixture is only a fallback for environments that loaded their cost map from a stale remote source. --------- Co-authored-by: Tai An Co-authored-by: Vincent Co-authored-by: Kris Xia Co-authored-by: d 🔹 Co-authored-by: Fabrizio Cafolla Co-authored-by: Filippo Menghi <113345637+Cyberfilo@users.noreply.github.com> Co-authored-by: Tom Denham Co-authored-by: escon1004 <70471150+escon1004@users.noreply.github.com> Co-authored-by: Divyansh Singhal <97736786+Divyansh8321@users.noreply.github.com> Co-authored-by: robin-fiddler Co-authored-by: Michael-RZ-Berri Co-authored-by: Michael Riad Zaky Co-authored-by: ryan-crabbe-berri Co-authored-by: Krrish Dholakia Co-authored-by: Sameer Kankute Co-authored-by: Cursor Co-authored-by: Yassin Kortam Co-authored-by: Federico Kamelhar Co-authored-by: Claude --- .github/workflows/codeql.yml | 28 + litellm/llms/custom_httpx/llm_http_handler.py | 54 +- litellm/llms/oci/chat/cohere.py | 386 ++++ litellm/llms/oci/chat/generic.py | 477 +++++ litellm/llms/oci/chat/transformation.py | 1585 +++++------------ litellm/llms/oci/common_utils.py | 552 +++++- litellm/llms/oci/embed/transformation.py | 417 ++--- litellm/main.py | 34 +- ...odel_prices_and_context_window_backup.json | 45 + litellm/router.py | 8 +- litellm/types/llms/oci.py | 99 +- litellm/utils.py | 8 +- model_prices_and_context_window.json | 204 ++- .../code_coverage_tests/recursive_detector.py | 3 + tests/integration/oci_proxy_test_config.yaml | 24 + tests/integration/test_oci_integration.py | 669 +++++++ .../integration/test_oci_proxy_integration.py | 274 +++ .../oci/chat/test_oci_chat_transformation.py | 10 +- .../oci/chat/test_oci_chat_transformation.py | 563 +++++- .../oci/chat/test_oci_cohere_tool_calls.py | 300 +++- .../llms/oci/chat/test_oci_generic_chat.py | 440 +++++ .../llms/oci/chat/test_oci_sse_splitter.py | 232 +++ .../oci/chat/test_oci_streaming_tool_calls.py | 153 +- .../embed/test_oci_embed_transformation.py | 406 +++++ .../llms/oci/embed/test_oci_embedding.py | 30 +- .../test_litellm/llms/oci/rerank/__init__.py | 0 .../llms/oci/test_oci_common_utils.py | 521 ++++++ .../llms/oci/test_oci_coverage_boost.py | 1152 ++++++++++++ 28 files changed, 7013 insertions(+), 1661 deletions(-) create mode 100644 litellm/llms/oci/chat/cohere.py create mode 100644 litellm/llms/oci/chat/generic.py create mode 100644 tests/integration/oci_proxy_test_config.yaml create mode 100644 tests/integration/test_oci_integration.py create mode 100644 tests/integration/test_oci_proxy_integration.py create mode 100644 tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py create mode 100644 tests/test_litellm/llms/oci/chat/test_oci_sse_splitter.py create mode 100644 tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py create mode 100644 tests/test_litellm/llms/oci/rerank/__init__.py create mode 100644 tests/test_litellm/llms/oci/test_oci_common_utils.py create mode 100644 tests/test_litellm/llms/oci/test_oci_coverage_boost.py diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e86fca17c7a3..babe3b629335 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -53,3 +53,31 @@ jobs: uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: category: "/language:${{ matrix.language }}" + output: sarif-results + upload: failure-only + + # py/weak-sensitive-data-hashing (CWE-328) fires on the OCI signing call at + # litellm/llms/oci/common_utils.py, which hashes the HTTP request body to + # produce the x-content-sha256 header required by the OCI HTTP signing spec — + # a content-integrity hash, not a password or secret hash. SHA-256 is mandated + # by Oracle for this header; see + # https://docs.oracle.com/en-us/iaas/Content/API/Concepts/signingrequests.htm + # The `usedforsecurity=False` flag on the hashlib.sha256 call already declares + # non-security intent, but CodeQL's taint flow still re-fires when callers + # further up the stack are modified. The suppression is scoped to this one + # file/rule pair via SARIF post-filtering so every other callsite of + # py/weak-sensitive-data-hashing in the repository continues to be analyzed. + - name: Filter SARIF (OCI sha256) + if: matrix.language == 'python' + uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1 + with: + patterns: | + -litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing + input: sarif-results/python.sarif + output: sarif-results/python.sarif + + - name: Upload SARIF + uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 + with: + sarif_file: sarif-results + category: "/language:${{ matrix.language }}" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 96fdf4494f94..090aac187b3c 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -890,6 +890,18 @@ def embedding( headers=headers, ) + # Some providers (e.g. OCI) require request signing after the body is built. + # The default BaseConfig.sign_request returns (headers, None) — a no-op for + # providers that don't need signing. + headers, signed_body = provider_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=data, + api_base=api_base, + api_key=api_key, + model=model, + ) + ## LOGGING logging_obj.pre_call( input=input, @@ -916,6 +928,7 @@ def embedding( client=client, optional_params=optional_params, litellm_params=litellm_params, + signed_body=signed_body, ) if client is None or not isinstance(client, HTTPHandler): @@ -926,12 +939,20 @@ def embedding( sync_httpx_client = client try: - response = sync_httpx_client.post( - url=api_base, - headers=headers, - data=json.dumps(data), - timeout=timeout, - ) + if signed_body is not None: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=signed_body, + timeout=timeout, + ) + else: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout, + ) except Exception as e: raise self._handle_error( e=e, @@ -964,6 +985,7 @@ async def aembedding( api_key: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + signed_body: Optional[bytes] = None, ) -> EmbeddingResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( @@ -974,12 +996,20 @@ async def aembedding( async_httpx_client = client try: - response = await async_httpx_client.post( - url=api_base, - headers=headers, - json=request_data, - timeout=timeout, - ) + if signed_body is not None: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=signed_body, + timeout=timeout, + ) + else: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=request_data, + timeout=timeout, + ) except Exception as e: raise self._handle_error(e=e, provider_config=provider_config) diff --git a/litellm/llms/oci/chat/cohere.py b/litellm/llms/oci/chat/cohere.py new file mode 100644 index 000000000000..ac92fd22aa86 --- /dev/null +++ b/litellm/llms/oci/chat/cohere.py @@ -0,0 +1,386 @@ +""" +OCI Generative AI — Cohere-specific chat transformation helpers. + +Handles message history building, tool definition adaptation, non-streaming +response parsing, and streaming chunk parsing for models served with +``apiFormat="COHERE"`` (e.g. ``cohere.command-*``). +""" + +import datetime +import json +from typing import Any, Dict, List, Optional + +import httpx +from pydantic import ValidationError + +from litellm.llms.oci.chat.generic import ( + _normalize_oci_finish_reason, + _synthesize_oci_tool_call_id, +) +from litellm.llms.oci.common_utils import ( + OCI_JSON_TO_PYTHON_TYPES, + OCIError, + enrich_cohere_param_description, + resolve_oci_schema_anyof, + resolve_oci_schema_refs, + sanitize_oci_schema, +) +from litellm.types.llms.oci import ( + CohereChatResult, + CohereMessage, + CohereParameterDefinition, + CohereStreamChunk, + CohereTool, + CohereToolCall, + CohereToolMessage, + CohereToolResult, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ( + Choices, + Delta, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) +from litellm.types.utils import Usage + + +def _extract_text_content(content: Any) -> str: + """Return the plain-text representation of a message content value.""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join( + item.get("text", "") + for item in content + if isinstance(item, dict) and item.get("type") == "text" + ) + return str(content) + + +def adapt_messages_to_cohere_standard( + messages: List[AllMessageValues], +) -> List[CohereMessage]: + """Build a Cohere ``chatHistory`` list from an OpenAI-format message array. + + - All messages except the *last user message* are included. The caller pulls + the last user message into the request's top-level ``message`` field, so + trailing tool results (the standard agentic continuation pattern) still + appear in ``chatHistory`` and reach the model. + - If no user message exists, every message is included (no slice). + - System messages must be filtered out by the caller (they are routed into + ``preambleOverride`` separately) — they are not represented in + ``chatHistory``. + - Tool results are expressed as OCI ``CohereToolMessage.toolResults`` entries, + with the originating call's name and parameters resolved from the preceding + assistant message via a ``tool_call_id`` lookup. + """ + # First pass: build tool_call_id → CohereToolCall so tool-result messages can + # reference the originating call by name and parameters. + tool_call_lookup: Dict[str, CohereToolCall] = {} + for msg in messages: + if msg.get("role") == "assistant": + tool_calls_raw: Any = msg.get("tool_calls") or [] + for tc in tool_calls_raw: + tc_id = tc.get("id", "") + raw_args: Any = tc.get("function", {}).get("arguments", "{}") + try: + params: Dict[str, Any] = ( + json.loads(raw_args) if isinstance(raw_args, str) else raw_args + ) + except json.JSONDecodeError: + params = {} + tool_call_lookup[tc_id] = CohereToolCall( + name=str(tc.get("function", {}).get("name", "")), + parameters=params, + ) + + last_user_index = next( + ( + i + for i in range(len(messages) - 1, -1, -1) + if messages[i].get("role") == "user" + ), + None, + ) + history_source = ( + messages + if last_user_index is None + else [m for i, m in enumerate(messages) if i != last_user_index] + ) + + chat_history: List[CohereMessage] = [] + for msg in history_source: + role = msg.get("role") + content = _extract_text_content(msg.get("content")) + + tool_calls: Optional[List[CohereToolCall]] = None + if role == "assistant" and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item] + tool_calls = [] + for tc in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item] + raw_arguments: Any = tc.get("function", {}).get("arguments", {}) + if isinstance(raw_arguments, str): + try: + arguments: Dict[str, Any] = json.loads(raw_arguments) + except json.JSONDecodeError: + arguments = {} + else: + arguments = raw_arguments + tool_calls.append( + CohereToolCall( + name=str(tc.get("function", {}).get("name", "")), + parameters=arguments, + ) + ) + + if role == "user": + chat_history.append(CohereMessage(role="USER", message=content)) + elif role == "assistant": + chat_history.append( + CohereMessage(role="CHATBOT", message=content, toolCalls=tool_calls) + ) + elif role == "tool": + tool_call_id = str(msg.get("tool_call_id", "") or "") + cohere_call = tool_call_lookup.get( + tool_call_id, CohereToolCall(name="", parameters={}) + ) + tool_result = CohereToolResult( + call=cohere_call, + outputs=[{"output": content}], + ) + # OpenAI emits one tool-role message per parallel tool call, but + # the OCI Cohere API expects all results from a single assistant + # turn to share one TOOL history entry with multiple toolResults. + # Merge consecutive tool messages so the model sees the parallel + # call/result pairing correctly during agentic loops. + if chat_history and isinstance(chat_history[-1], CohereToolMessage): + chat_history[-1].toolResults.append(tool_result) + else: + chat_history.append(CohereToolMessage(toolResults=[tool_result])) + + return chat_history + + +def adapt_tool_definitions_to_cohere_standard( + tools: List[Dict[str, Any]], +) -> List[CohereTool]: + """Adapt OpenAI-format tool definitions to the OCI Cohere format. + + - Resolves ``$ref``/``$defs`` and ``anyOf`` patterns that OCI rejects. + - Maps JSON Schema type names to Python type names (``"string"`` → ``"str"``). + - Embeds unsupported constraints (enum, format, range, pattern) into the + parameter description so the model can still see them. + """ + cohere_tools = [] + for tool in tools: + function_def = tool.get("function", {}) + raw_params = function_def.get("parameters", {}) + + resolved = sanitize_oci_schema( + resolve_oci_schema_anyof(resolve_oci_schema_refs(raw_params)) + ) + properties = resolved.get("properties", {}) + required = resolved.get("required", []) + + parameter_definitions = {} + for param_name, param_schema in properties.items(): + json_type = param_schema.get("type", "string") + python_type = OCI_JSON_TO_PYTHON_TYPES.get(json_type, json_type) + parameter_definitions[param_name] = CohereParameterDefinition( + description=enrich_cohere_param_description( + param_schema.get("description", ""), param_schema + ), + type=python_type, + isRequired=param_name in required, + ) + + cohere_tools.append( + CohereTool( + name=function_def.get("name", ""), + description=function_def.get("description", ""), + parameterDefinitions=parameter_definitions, + ) + ) + + return cohere_tools + + +def handle_cohere_response( + json_response: dict, + model: str, + model_response: ModelResponse, + raw_response: httpx.Response, +) -> ModelResponse: + """Parse a non-streaming Cohere OCI response into a LiteLLM ModelResponse.""" + try: + cohere_response = CohereChatResult(**json_response) + except (TypeError, ValidationError) as e: + raise OCIError( + message=f"Response cannot be casted to CohereChatResult: {str(e)}", + status_code=raw_response.status_code, + ) + + model_response.model = model + model_response.created = int(datetime.datetime.now().timestamp()) + + response_text = cohere_response.chatResponse.text + finish_reason = _normalize_oci_finish_reason( + cohere_response.chatResponse.finishReason + ) + + tool_calls: Optional[List[Dict[str, Any]]] = None + if cohere_response.chatResponse.toolCalls: + tool_calls = [ + { + "id": _synthesize_oci_tool_call_id( + i, tc.name, json.dumps(tc.parameters, sort_keys=True) + ), + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps(tc.parameters), + }, + } + for i, tc in enumerate(cohere_response.chatResponse.toolCalls) + ] + + content: Optional[str] = response_text if response_text else None + + # Only include ``tool_calls`` in the message dict when actually present. + # Passing an explicit ``None`` would let downstream consumers that key off + # ``"tool_calls" in message`` (rather than truthiness) incorrectly conclude + # that tool calls were attempted. Matches the generic handler's behaviour, + # which only sets ``message.tool_calls`` when tool calls are present. + message: Dict[str, Any] = {"role": "assistant", "content": content} + if tool_calls is not None: + message["tool_calls"] = tool_calls + + model_response.choices = [ + Choices( + index=0, + message=message, + finish_reason=finish_reason, + ) + ] + + usage_info = cohere_response.chatResponse.usage + if usage_info is not None: + model_response.usage = Usage( # type: ignore[attr-defined] + prompt_tokens=usage_info.promptTokens, + completion_tokens=usage_info.completionTokens, + total_tokens=usage_info.totalTokens, + ) + else: + model_response.usage = Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) # type: ignore[attr-defined] + + return model_response + + +def handle_cohere_stream_chunk( + dict_chunk: dict, + prior_tool_calls_emitted: bool = False, + prior_text_emitted: bool = False, +) -> ModelResponseStream: + """Parse a single Cohere SSE chunk into a LiteLLM ModelResponseStream. + + ``prior_tool_calls_emitted`` lets the caller signal whether tool calls + were already emitted in earlier chunks of the same stream. When set, the + terminal consolidation chunk's tool calls are suppressed (they would + duplicate prior deltas); otherwise they are passed through so a stream + that delivers tool calls only on the terminal chunk doesn't silently + drop them. + + ``prior_text_emitted`` plays the analogous role for the ``text`` field: + when set, the terminal consolidation chunk's ``text`` is suppressed + (it would re-emit the full assembled response on top of prior deltas); + when unset (e.g. a degenerate stream that delivers the entire response + in a single SSE event carrying both ``chatHistory`` and ``finishReason``), + the text is passed through so the response content isn't silently lost. + """ + try: + typed_chunk = CohereStreamChunk(**dict_chunk) + except (TypeError, ValidationError) as e: + raise OCIError( + status_code=500, + message=f"Chunk cannot be parsed as CohereStreamChunk: {str(e)}", + ) + + if typed_chunk.index is None: + typed_chunk.index = 0 + + # OCI Cohere's terminal SSE event re-sends the full assembled response in + # `text` alongside a populated `chatHistory` and a non-null `finishReason`. + # Emitting that text would concatenate the whole response onto the + # already-streamed deltas. We require both signals to be present so that a + # future API change which adds `chatHistory` to intermediate chunks (or a + # rare early-populated case) doesn't silently drop legitimate token deltas. + is_terminal_consolidation = ( + typed_chunk.chatHistory is not None and typed_chunk.finishReason is not None + ) + # On non-terminal text-free chunks (e.g. tool-call-only or keep-alive + # chunks) emit ``content=None`` rather than ``content=""`` so downstream + # stream-mergers that distinguish "no text in this delta" from "an + # explicitly empty text delta" behave correctly. + # + # We only suppress the terminal chunk's ``text`` when the caller has + # confirmed that text deltas were already emitted earlier — otherwise + # (e.g. a degenerate stream that delivers the whole response in a + # single SSE event), passing it through is the only chance to surface it. + text: Optional[str] = ( + None if (is_terminal_consolidation and prior_text_emitted) else typed_chunk.text + ) + + # Tool calls on the terminal consolidation chunk (whether from + # `typed_chunk.toolCalls` or from `chatHistory`) typically restate what + # was already streamed in intermediate chunks. Re-emitting them would + # mint fresh `uuid4` IDs and cause downstream consumers to execute each + # tool call twice. We only suppress when the caller has confirmed that + # tool calls were already emitted earlier — otherwise (e.g. a short + # response that delivers tool calls exclusively on the terminal chunk), + # passing them through is the only chance to surface them. + cohere_tool_calls = ( + None + if (is_terminal_consolidation and prior_tool_calls_emitted) + else typed_chunk.toolCalls + ) + + tool_calls: Optional[List[Dict[str, Any]]] = None + if cohere_tool_calls: + tool_calls = [ + { + # Cohere protocol has no tool-call id, so we synthesize one + # deterministically from the call's content/position. A random + # uuid4 per chunk would cause downstream stream-mergers to + # treat each chunk as a distinct tool call. + "id": _synthesize_oci_tool_call_id( + i, tc.name, json.dumps(tc.parameters, sort_keys=True) + ), + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps(tc.parameters), + }, + } + for i, tc in enumerate(cohere_tool_calls) + ] + + finish_reason = _normalize_oci_finish_reason(typed_chunk.finishReason) + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=typed_chunk.index, + delta=Delta( + content=text, + tool_calls=tool_calls, + provider_specific_fields=None, + thinking_blocks=None, + reasoning_content=None, + ), + finish_reason=finish_reason, + ) + ] + ) diff --git a/litellm/llms/oci/chat/generic.py b/litellm/llms/oci/chat/generic.py new file mode 100644 index 000000000000..2cc1ac77a40d --- /dev/null +++ b/litellm/llms/oci/chat/generic.py @@ -0,0 +1,477 @@ +""" +OCI Generative AI — Generic-format chat transformation helpers. + +Handles message building, tool definition adaptation, non-streaming response +parsing, and streaming chunk parsing for models served with +``apiFormat="GENERIC"`` (e.g. Meta Llama, xAI Grok, Google Gemini). +""" + +import datetime +import hashlib +from typing import Any, Dict, List, Optional, Union + +import httpx +from pydantic import ValidationError + +from litellm.llms.oci.common_utils import ( + OCIError, + resolve_oci_schema_anyof, + resolve_oci_schema_refs, + sanitize_oci_schema, +) +from litellm.types.llms.oci import ( + OCICompletionResponse, + OCIContentPartUnion, + OCIImageContentPart, + OCIImageUrl, + OCIMessage, + OCIRoles, + OCIStreamChunk, + OCITextContentPart, + OCIToolCall, + OCIToolDefinition, + OCIVendors, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ( + Delta, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) +from litellm.types.utils import ChatCompletionMessageToolCall, Usage + +# Maps OpenAI role names to OCI GENERIC role names. +open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = { + "system": "SYSTEM", + "user": "USER", + "assistant": "ASSISTANT", + "tool": "TOOL", +} + + +# --------------------------------------------------------------------------- +# Message building +# --------------------------------------------------------------------------- + + +def adapt_messages_to_generic_oci_standard_content_message( + role: str, content: Union[str, list] +) -> OCIMessage: + """Convert a plain-text or multipart content message to OCI format.""" + new_content: List[OCIContentPartUnion] = [] + if isinstance(content, str): + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=None, + ) + + for content_item in content: + if not isinstance(content_item, dict): + raise OCIError( + status_code=400, message="Each content item must be a dictionary" + ) + + item_type = content_item.get("type") + if not isinstance(item_type, str): + raise OCIError( + status_code=400, + message="Each content item must have a string `type` field", + ) + if item_type not in ["text", "image_url"]: + raise OCIError( + status_code=400, + message=f"Content type `{item_type}` is not supported by OCI", + ) + + if item_type == "text": + text = content_item.get("text") + if not isinstance(text, str): + raise OCIError( + status_code=400, + message="Content item of type `text` must have a string `text` field", + ) + new_content.append(OCITextContentPart(text=text)) + + elif item_type == "image_url": + image_url = content_item.get("image_url") + if isinstance(image_url, dict): + image_url = image_url.get("url") + if not isinstance(image_url, str): + raise OCIError( + status_code=400, + message="Prop `image_url` must be a string or an object with a `url` property", + ) + new_content.append(OCIImageContentPart(imageUrl=OCIImageUrl(url=image_url))) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=new_content, + toolCalls=None, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_call( + role: str, tool_calls: list +) -> OCIMessage: + """Convert an assistant tool-call message to OCI format.""" + tool_calls_formatted = [] + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + raise OCIError( + status_code=400, message="Each tool call must be a dictionary" + ) + if tool_call.get("type") != "function": + raise OCIError( + status_code=400, message="OCI only supports function tool calls" + ) + + tool_call_id = tool_call.get("id") + if not isinstance(tool_call_id, str): + raise OCIError(status_code=400, message="Tool call `id` must be a string") + + tool_function = tool_call.get("function") + if not isinstance(tool_function, dict): + raise OCIError( + status_code=400, message="Tool call `function` must be a dictionary" + ) + + function_name = tool_function.get("name") + if not isinstance(function_name, str): + raise OCIError( + status_code=400, message="Tool call `function.name` must be a string" + ) + + arguments = tool_call["function"].get("arguments", "{}") + if not isinstance(arguments, str): + raise OCIError( + status_code=400, + message="Tool call `function.arguments` must be a JSON string", + ) + + tool_calls_formatted.append( + OCIToolCall( + id=tool_call_id, + type="FUNCTION", + name=function_name, + arguments=arguments, + ) + ) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=None, + toolCalls=tool_calls_formatted, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_response( + role: str, tool_call_id: str, content: str +) -> OCIMessage: + """Convert a tool-result message to OCI format.""" + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=tool_call_id, + ) + + +def adapt_messages_to_generic_oci_standard( + messages: List[AllMessageValues], +) -> List[OCIMessage]: + """Convert an OpenAI-format message array to OCI GENERIC format.""" + new_messages = [] + for message in messages: + role = message["role"] + content = message.get("content") + tool_calls = message.get("tool_calls") + tool_call_id = message.get("tool_call_id") + + if role == "assistant" and tool_calls is not None: + if not isinstance(tool_calls, list): + raise OCIError( + status_code=400, message="Message `tool_calls` must be a list" + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) + ) + + elif role in ["system", "user", "assistant"] and content is not None: + if not isinstance(content, (str, list)): + raise OCIError( + status_code=400, + message="Message `content` must be a string or list of content parts", + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_content_message(role, content) + ) + + elif role == "tool": + if not isinstance(tool_call_id, str): + raise OCIError( + status_code=400, + message="Tool result message must have a string `tool_call_id`", + ) + if not isinstance(content, str): + raise OCIError( + status_code=400, + message="Tool result message `content` must be a string", + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_response( + role, tool_call_id, content + ) + ) + + return new_messages + + +# --------------------------------------------------------------------------- +# Tool definition adaptation +# --------------------------------------------------------------------------- + + +def adapt_tool_definition_to_oci_standard( + tools: List[Dict], vendor: OCIVendors +) -> List[OCIToolDefinition]: + """Convert OpenAI-format tool definitions to OCI GENERIC format. + + Resolves ``$ref``/``$defs`` and ``anyOf`` that the OCI endpoint rejects. + """ + new_tools = [] + for tool in tools: + if tool["type"] != "function": + raise OCIError(status_code=400, message="OCI only supports function tools") + + tool_function = tool.get("function") + if not isinstance(tool_function, dict): + raise OCIError( + status_code=400, message="Tool `function` must be a dictionary" + ) + + raw_params = tool_function.get("parameters", {}) + resolved_params = sanitize_oci_schema( + resolve_oci_schema_anyof(resolve_oci_schema_refs(raw_params)) + ) + + new_tools.append( + OCIToolDefinition( + type="FUNCTION", + name=tool_function.get("name"), + description=tool_function.get("description", ""), + parameters=resolved_params, + ) + ) + + return new_tools + + +def _normalize_oci_finish_reason(raw: Optional[str]) -> Optional[str]: + """Map an OCI-specific finish reason to its OpenAI-standard equivalent. + + OCI emits ``COMPLETE`` / ``MAX_TOKENS`` / ``TOOL_CALL(S)`` plus a long tail + of error/cancel reasons (``ERROR``, ``ERROR_TOXIC``, ``ERROR_LIMIT``, + ``USER_CANCEL``, ``CONTENT_FILTERED``, ``CANCELLED``, ...). The OpenAI + spec only defines ``stop`` / ``length`` / ``tool_calls`` / ... — anything + else is collapsed to ``"stop"`` so downstream consumers switching on + ``finish_reason`` keep working. A ``None`` input passes through unchanged. + """ + if raw is None: + return None + if raw == "COMPLETE": + return "stop" + if raw == "MAX_TOKENS": + return "length" + if raw in ("TOOL_CALL", "TOOL_CALLS"): + return "tool_calls" + return "stop" + + +def _synthesize_oci_tool_call_id(position: int, name: str, arguments: str) -> str: + """Deterministic synthetic tool-call id derived from chunk content. + + Used as a fallback when OCI omits ``id`` (always the case for the OCI + Cohere protocol, occasionally the case for OCI GENERIC streaming chunks). + A random ``uuid4`` per chunk would cause downstream stream-merging + consumers — which key off the tool-call ``id`` — to treat re-emissions of + the same logical call (e.g. terminal consolidation chunks, retries) as + distinct calls. A content-derived digest stays stable across identical + re-emissions while differing across truly distinct calls. + """ + digest = hashlib.sha256( + f"{position}|{name}|{arguments}".encode("utf-8"), + usedforsecurity=False, + ).hexdigest()[:24] + return f"call_{digest}" + + +def adapt_tools_to_openai_standard( + tools: List[OCIToolCall], +) -> List[ChatCompletionMessageToolCall]: + """Convert OCI tool-call objects in a response to the OpenAI format.""" + return [ + ChatCompletionMessageToolCall( + id=tool.id or _synthesize_oci_tool_call_id(i, tool.name, tool.arguments), + type="function", + function={"name": tool.name, "arguments": tool.arguments}, + ) + for i, tool in enumerate(tools) + ] + + +# --------------------------------------------------------------------------- +# Response parsing +# --------------------------------------------------------------------------- + + +def handle_generic_response( + json_data: dict, + model: str, + model_response: ModelResponse, + raw_response: httpx.Response, +) -> ModelResponse: + """Parse a non-streaming GENERIC OCI response into a LiteLLM ModelResponse.""" + try: + completion_response = OCICompletionResponse(**json_data) + except (TypeError, ValidationError) as e: + raise OCIError( + message=f"Response cannot be casted to OCICompletionResponse: {str(e)}", + status_code=raw_response.status_code, + ) + + iso_str = completion_response.chatResponse.timeCreated + dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00")) + model_response.created = int(dt.timestamp()) + model_response.model = completion_response.modelId + + if not completion_response.chatResponse.choices: + raise OCIError( + message="OCI response contained no choices", + status_code=raw_response.status_code, + ) + + response_choice = completion_response.chatResponse.choices[0] + message = model_response.choices[0].message # type: ignore + response_message = response_choice.message + if response_message is not None: + if response_message.content: + # Concatenate all text parts — matches the streaming handler, which + # iterates the full content array. Skips non-text parts (e.g. image + # parts) so a leading non-text part doesn't suppress trailing text. + text: Optional[str] = None + for item in response_message.content: + if isinstance(item, OCITextContentPart): + text = (text or "") + item.text + if text is not None: + message.content = text + if response_message.toolCalls: + message.tool_calls = adapt_tools_to_openai_standard( + response_message.toolCalls + ) + + model_response.choices[0].finish_reason = _normalize_oci_finish_reason( # type: ignore[union-attr,assignment] + response_choice.finishReason + ) + + oci_usage = completion_response.chatResponse.usage + reasoning_tokens: Optional[int] = None + if ( + oci_usage.completionTokensDetails + and oci_usage.completionTokensDetails.reasoningTokens is not None + ): + reasoning_tokens = oci_usage.completionTokensDetails.reasoningTokens + model_response.usage = Usage( # type: ignore[attr-defined] + prompt_tokens=oci_usage.promptTokens, + completion_tokens=oci_usage.completionTokens or 0, + total_tokens=oci_usage.totalTokens, + reasoning_tokens=reasoning_tokens, + ) + + return model_response + + +def handle_generic_stream_chunk(dict_chunk: dict) -> ModelResponseStream: + """Parse a single GENERIC SSE chunk into a LiteLLM ModelResponseStream.""" + # OCI streams tool calls progressively — early chunks may omit required fields. + if dict_chunk.get("message") and dict_chunk["message"].get("toolCalls"): + for tool_call in dict_chunk["message"]["toolCalls"]: + tool_call.setdefault("arguments", "") + tool_call.setdefault("id", "") + tool_call.setdefault("name", "") + + try: + typed_chunk = OCIStreamChunk(**dict_chunk) + except (TypeError, ValidationError) as e: + raise OCIError( + status_code=500, + message=f"Chunk cannot be parsed as OCIStreamChunk: {str(e)}", + ) + + if typed_chunk.index is None: + typed_chunk.index = 0 + + # Emit ``content=None`` rather than ``content=""`` on chunks with no text + # parts (e.g. tool-call-only or keep-alive chunks) so downstream + # stream-mergers that distinguish "no text in this delta" from "an + # explicitly empty text delta" behave correctly. + text: Optional[str] = None + if typed_chunk.message and typed_chunk.message.content: + for item in typed_chunk.message.content: + if isinstance(item, OCITextContentPart): + text = (text or "") + item.text + elif isinstance(item, OCIImageContentPart): + raise OCIError( + status_code=500, + message="OCI returned image content in a streaming response — not supported", + ) + else: + raise OCIError( + status_code=500, + message=f"Unsupported content type in OCI streaming response: {item.type}", + ) + + # Build plain tool-call dicts inline (matching the shape produced by + # ``handle_cohere_stream_chunk``) rather than calling + # ``adapt_tools_to_openai_standard`` and ``model_dump``-ing the typed + # objects. Both code paths feed ``Delta.tool_calls``, so emitting the + # same minimal ``{"id", "type", "function": {"name", "arguments"}}`` + # shape keeps downstream stream-mergers behaving identically across + # GENERIC and Cohere chunks. + tool_calls: Optional[List[Dict[str, Any]]] = None + if typed_chunk.message and typed_chunk.message.toolCalls: + tool_calls = [ + { + "id": tc.id or _synthesize_oci_tool_call_id(i, tc.name, tc.arguments), + "type": "function", + "function": { + "name": tc.name, + "arguments": tc.arguments, + }, + } + for i, tc in enumerate(typed_chunk.message.toolCalls) + ] + + finish_reason: Optional[str] = _normalize_oci_finish_reason( + typed_chunk.finishReason + ) + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=typed_chunk.index, + delta=Delta( + content=text, + tool_calls=tool_calls, + provider_specific_fields=None, + thinking_blocks=None, + reasoning_content=None, + ), + finish_reason=finish_reason, + ) + ] + ) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 62104e921a4e..f050f9eea36e 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -1,20 +1,26 @@ -import base64 -import datetime -import hashlib +""" +OCI Generative AI — chat transformation orchestrator. + +This module wires together the Cohere-specific and Generic-model helpers to +implement the LiteLLM BaseConfig interface. Heavy-lifting lives in: + + - :mod:`litellm.llms.oci.chat.cohere` — Cohere message/tool/response logic + - :mod:`litellm.llms.oci.chat.generic` — Generic message/tool/response logic + - :mod:`litellm.llms.oci.common_utils` — auth, signing, schema utilities +""" + import json -from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, AsyncIterator, Dict, + Iterator, List, Optional, - Protocol, Tuple, Union, ) -from urllib.parse import urlparse import httpx @@ -28,43 +34,43 @@ get_async_httpx_client, version, ) -from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.chat.cohere import ( + _extract_text_content, + adapt_messages_to_cohere_standard, + adapt_tool_definitions_to_cohere_standard, + handle_cohere_response, + handle_cohere_stream_chunk, +) +from litellm.llms.oci.chat.generic import ( + adapt_messages_to_generic_oci_standard, + adapt_tool_definition_to_oci_standard, + handle_generic_response, + handle_generic_stream_chunk, +) +from litellm.llms.oci.common_utils import ( + OCI_API_VERSION, + OCIError, + OCIRequestWrapper, # re-exported for backwards compatibility + get_oci_base_url, + resolve_oci_credentials, + sign_oci_request, + validate_oci_environment, +) from litellm.types.llms.oci import ( CohereChatRequest, - CohereMessage, - CohereChatResult, - CohereParameterDefinition, - CohereStreamChunk, - CohereTool, - CohereToolCall, OCIChatRequestPayload, OCICompletionPayload, - OCICompletionResponse, - OCIContentPartUnion, - OCIImageContentPart, - OCIImageUrl, - OCIMessage, - OCIRoles, OCIServingMode, - OCIStreamChunk, - OCITextContentPart, - OCIToolCall, - OCIToolDefinition, OCIVendors, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( - Delta, LlmProviders, ModelResponse, ModelResponseStream, - StreamingChoices, -) -from litellm.utils import ( - ChatCompletionMessageToolCall, - CustomStreamWrapper, - Usage, ) +from litellm.utils import supports_reasoning +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -74,142 +80,157 @@ LiteLLMLoggingObj = Any -class OCISignerProtocol(Protocol): - """ - Protocol for OCI request signers (e.g., oci.signer.Signer). - - This protocol defines the interface expected for OCI SDK signer objects. - Compatible with the OCI Python SDK's Signer class. - - See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html - """ - - def do_request_sign( - self, request: Any, *, enforce_content_headers: bool = False - ) -> None: - """ - Sign an HTTP request by adding authentication headers. - - Args: - request: Request object with method, url, headers, body, and path_url attributes - enforce_content_headers: Whether to enforce content-type and content-length headers - """ - ... +# Streaming timeout — generous because OCI models may need to warm up on first request +STREAMING_TIMEOUT = 60 * 5 -@dataclass -class OCIRequestWrapper: - """ - Wrapper for HTTP requests compatible with OCI signer interface. +def _model_uses_max_completion_tokens(model: str) -> bool: + """Return True for OCI-hosted models that require ``maxCompletionTokens``. - This class wraps request data in a format compatible with OCI SDK signers, - which expect objects with method, url, headers, body, and path_url attributes. + Reasoning models on OCI (e.g. the OpenAI GPT-5 family) reject ``maxTokens`` + with HTTP 400 and require ``maxCompletionTokens`` per OpenAI's reasoning-API + convention. Driven by ``supports_reasoning`` in + ``model_prices_and_context_window.json`` so new model families are picked + up via a catalog update rather than a code change. """ + if not model: + return False + name = model[4:] if model.lower().startswith("oci/") else model + return supports_reasoning(model=name, custom_llm_provider="oci") - method: str - url: str - headers: dict - body: bytes - - @property - def path_url(self) -> str: - """Returns the path + query string for OCI signing.""" - parsed_url = urlparse(self.url) - return parsed_url.path + ("?" + parsed_url.query if parsed_url.query else "") +def _iter_sse_events(stream: Iterator[str]) -> Iterator[str]: + """Yield one ``data:`` SSE line at a time from a sync text stream. -def sha256_base64(data: bytes) -> str: - digest = hashlib.sha256(data).digest() - return base64.b64encode(digest).decode() - - -def build_signature_string(method, path, headers, signed_headers): - lines = [] - for header in signed_headers: - if header == "(request-target)": - value = f"{method.lower()} {path}" + The OCI streaming endpoint does not align SSE event boundaries with HTTP + read boundaries. A single read may carry multiple events, a single event + may straddle two reads, and some events arrive separated by only ``\\n`` + instead of ``\\n\\n``. This helper buffers across reads and yields each + complete ``data:`` line so JSON parsing downstream never sees a partial + payload. + """ + buffer = "" + for item in stream: + buffer += item + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + stripped = line.strip() + if stripped.startswith("data:"): + yield stripped + stripped = buffer.strip() + if stripped.startswith("data:"): + yield stripped + + +async def _aiter_sse_events(stream: AsyncIterator[str]) -> AsyncIterator[str]: + """Async twin of :func:`_iter_sse_events`.""" + buffer = "" + async for item in stream: + buffer += item + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + stripped = line.strip() + if stripped.startswith("data:"): + yield stripped + stripped = buffer.strip() + if stripped.startswith("data:"): + yield stripped + + +def _normalize_tool_choice(selected_params: Dict) -> None: + tc = selected_params.get("toolChoice") + if tc is None: + return + if isinstance(tc, str): + tc_map = { + "auto": {"type": "AUTO"}, + "none": {"type": "NONE"}, + "required": {"type": "REQUIRED"}, + "any": {"type": "REQUIRED"}, + } + selected_params["toolChoice"] = tc_map.get( + tc.lower(), {"type": "FUNCTION", "name": tc} + ) + return + if isinstance(tc, dict): + raw_type = tc.get("type") + if not isinstance(raw_type, str): + raise OCIError( + status_code=400, + message=f"Invalid tool_choice for OCI: missing or non-string 'type' in {tc!r}", + ) + upper = raw_type.upper() + if upper == "FUNCTION": + fn = tc.get("function") + name = fn.get("name") if isinstance(fn, dict) else tc.get("name") + if not (isinstance(name, str) and name): + raise OCIError( + status_code=400, + message="Invalid tool_choice for OCI: 'FUNCTION' type requires a non-empty function name", + ) + selected_params["toolChoice"] = {"type": "FUNCTION", "name": name} + elif upper in {"AUTO", "NONE", "REQUIRED"}: + selected_params["toolChoice"] = {"type": upper} else: - value = headers[header] - lines.append(f"{header}: {value}") - return "\n".join(lines) - - -def load_private_key_from_str(key_str: str): - try: - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric import rsa - except ImportError as e: - raise ImportError( - "cryptography package is required for OCI authentication. " - "Please install it with: pip install cryptography" - ) from e - - key = serialization.load_pem_private_key( - key_str.encode("utf-8"), - password=None, + raise OCIError( + status_code=400, + message=( + f"Invalid tool_choice for OCI: unsupported type {raw_type!r}; " + "expected one of 'FUNCTION', 'AUTO', 'NONE', 'REQUIRED'" + ), + ) + return + raise OCIError( + status_code=400, + message=( + f"Invalid tool_choice for OCI: expected str or dict, got " + f"{type(tc).__name__}" + ), ) - if not isinstance(key, rsa.RSAPrivateKey): - raise TypeError( - "The provided private key is not an RSA key, which is required for OCI signing." - ) - return key - -def load_private_key_from_file(file_path: str): - """Loads a private key from a file path""" - try: - with open(file_path, "r", encoding="utf-8") as f: - key_str = f.read().strip() - except FileNotFoundError: - raise FileNotFoundError(f"Private key file not found: {file_path}") - except OSError as e: - raise OSError(f"Failed to read private key file '{file_path}': {e}") from e - if not key_str: - raise ValueError(f"Private key file is empty: {file_path}") - - return load_private_key_from_str(key_str) +def _normalize_response_format(selected_params: Dict, vendor: OCIVendors) -> None: + rf = selected_params.get("responseFormat") + if not isinstance(rf, dict) or "type" not in rf: + return + rf_payload = dict(rf) + selected_params["responseFormat"] = rf_payload + response_type = rf_payload["type"] + if "json_schema" in rf_payload: + raw_schema = rf_payload.pop("json_schema") + rf_payload["jsonSchema"] = ( + dict(raw_schema) if isinstance(raw_schema, dict) else raw_schema + ) + if vendor == OCIVendors.COHERE: + rf_payload["type"] = response_type + else: + fmt = response_type.upper() + rf_payload["type"] = "JSON_OBJECT" if fmt == "JSON" else fmt def get_vendor_from_model(model: str) -> OCIVendors: - """ - Extracts the vendor from the model name. + """Return the OCI vendor enum for a model name. - OCI GenAI API uses two apiFormat values: - - "COHERE" for Cohere models (command-r, command-a, etc.) - - "GENERIC" for all other models (Meta Llama, xAI Grok, Google Gemini, etc.) + OCI GenAI uses two ``apiFormat`` values: - Args: - model (str): The model name (e.g., "cohere.command-a-03-2025", "meta.llama-3.3-70b-instruct"). - Returns: - OCIVendors: The vendor enum value. + - ``"COHERE"`` for Cohere models (``cohere.*``) + - ``"GENERIC"`` for all others (Meta Llama, xAI Grok, Google Gemini, …) """ - vendor = model.split(".")[0].lower() + name = model[4:] if model.lower().startswith("oci/") else model + vendor = name.split(".")[0].lower() if vendor == "cohere": return OCIVendors.COHERE - else: - return OCIVendors.GENERIC - - -# 5 minute timeout (models may need to load) -STREAMING_TIMEOUT = 60 * 5 + return OCIVendors.GENERIC class OCIChatConfig(BaseConfig): - """ - Configuration class for OCI's API interface. - """ + """LiteLLM BaseConfig implementation for OCI Generative AI chat.""" - def __init__( - self, - ) -> None: - locals_ = locals().copy() - for key, value in locals_.items(): - if key != "self" and value is not None: - setattr(self.__class__, key, value) - # mark the class as using a custom stream wrapper because the default only iterates on lines - setattr(self.__class__, "has_custom_stream_wrapper", True) + @property + def has_custom_stream_wrapper(self) -> bool: + return True + def __init__(self) -> None: self.openai_to_oci_generic_param_map = { "stream": "isStream", "max_tokens": "maxTokens", @@ -221,6 +242,7 @@ def __init__( "logit_bias": "logitBias", "n": "numGenerations", "presence_penalty": "presencePenalty", + "reasoning_effort": "reasoningEffort", "seed": "seed", "stop": "stop", "tool_choice": "toolChoice", @@ -239,25 +261,43 @@ def __init__( "response_format": "responseFormat", } - # Cohere and Gemini use the same parameter mapping as GENERIC - self.openai_to_oci_cohere_param_map = ( - self.openai_to_oci_generic_param_map.copy() - ) + # Cohere param map differs from GENERIC in three ways: + # - tool_choice is unsupported + # - stop sequences key is "stopSequences" not "stop" + # - n (numGenerations) is GENERIC-only + # The unsupported keys are kept in the map with value ``False`` so + # ``map_openai_params`` either drops them (under drop_params) or raises + # a clear error, rather than silently passing them through. + self.openai_to_oci_cohere_param_map = { + k: ("stopSequences" if k == "stop" else v) + for k, v in self.openai_to_oci_generic_param_map.items() + } + self.openai_to_oci_cohere_param_map["tool_choice"] = False + self.openai_to_oci_cohere_param_map["n"] = False + # ``top_k`` is not a standard OpenAI param, but Cohere's chat request + # accepts ``topK`` and LiteLLM commonly forwards ``top_k`` as a + # passthrough param. Cohere-only — ``OCIChatRequestPayload`` (GENERIC) + # has no ``topK`` field. + self.openai_to_oci_cohere_param_map["top_k"] = "topK" + # OCI Cohere models are not reasoning models; mark reasoning_effort + # explicitly unsupported so callers either get a clear error or have + # the param dropped under drop_params, rather than silently passing + # through and tripping Pydantic validation on CohereChatRequest. + self.openai_to_oci_cohere_param_map["reasoning_effort"] = False + # CohereChatRequest has no logProbs/logitBias fields, so passing these + # through would be silently dropped by Pydantic. Mark them unsupported + # so get_supported_openai_params doesn't advertise them and callers + # get a clear error (or drop_params behaviour) instead. + self.openai_to_oci_cohere_param_map["logprobs"] = False + self.openai_to_oci_cohere_param_map["logit_bias"] = False def get_supported_openai_params(self, model: str) -> List[str]: - supported_params = [] - vendor = get_vendor_from_model(model) - if vendor == OCIVendors.COHERE: - open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map - open_ai_to_oci_param_map.pop("tool_choice") - open_ai_to_oci_param_map.pop("max_retries") - else: - open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map - for key, value in open_ai_to_oci_param_map.items(): - if value: - supported_params.append(key) - - return supported_params + param_map = ( + self.openai_to_oci_cohere_param_map + if get_vendor_from_model(model) == OCIVendors.COHERE + else self.openai_to_oci_generic_param_map + ) + return [key for key, value in param_map.items() if value] def map_openai_params( self, @@ -268,238 +308,34 @@ def map_openai_params( ) -> dict: adapted_params = {} vendor = get_vendor_from_model(model) - if vendor == OCIVendors.COHERE: - open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map - else: - open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map - - all_params = {**non_default_params, **optional_params} - - for key, value in all_params.items(): - alias = open_ai_to_oci_param_map.get(key) + param_map = ( + self.openai_to_oci_cohere_param_map + if vendor == OCIVendors.COHERE + else self.openai_to_oci_generic_param_map + ) + for key, value in {**non_default_params, **optional_params}.items(): + alias = param_map.get(key) if alias is False: - # Workaround for mypy issue if drop_params or litellm.drop_params: continue - raise Exception(f"param `{key}` is not supported on OCI") - + raise OCIError( + status_code=400, + message=f"param `{key}` is not supported on OCI", + ) if alias is None: adapted_params[key] = value continue - adapted_params[alias] = value - + # Preserve the original OpenAI ``response_format`` key alongside the + # OCI-mapped ``responseFormat`` so downstream litellm framework code + # (e.g. ``json_mode`` detection, logging) that inspects + # ``optional_params["response_format"]`` continues to work. if alias == "responseFormat": adapted_params["response_format"] = value return adapted_params - def _sign_with_oci_signer( - self, - headers: dict, - optional_params: dict, - request_data: dict, - api_base: str, - ) -> Tuple[dict, bytes]: - """ - Sign request using OCI SDK Signer object. - - Args: - headers: Request headers to be signed - optional_params: Optional parameters including oci_signer - request_data: The request body dict to be sent in HTTP request - api_base: The complete URL for the HTTP request - - Returns: - Tuple of (signed_headers, encoded_body) - - Raises: - OCIError: If signing fails - ValueError: If HTTP method is unsupported - """ - oci_signer = optional_params.get("oci_signer") - body = json.dumps(request_data).encode("utf-8") - method = str(optional_params.get("method", "POST")).upper() - - if method not in ["POST", "GET", "PUT", "DELETE", "PATCH"]: - raise ValueError(f"Unsupported HTTP method: {method}") - - prepared_headers = headers.copy() - prepared_headers.setdefault("content-type", "application/json") - prepared_headers.setdefault("content-length", str(len(body))) - - request_wrapper = OCIRequestWrapper( - method=method, url=api_base, headers=prepared_headers, body=body - ) - - if oci_signer is None: - raise ValueError( - "oci_signer cannot be None when calling _sign_with_oci_signer" - ) - - try: - oci_signer.do_request_sign(request_wrapper, enforce_content_headers=True) - except Exception as e: - raise OCIError( - status_code=500, - message=( - f"Failed to sign request with provided oci_signer: {str(e)}. " - "The signer must implement the OCI SDK Signer interface with a " - "do_request_sign(request, enforce_content_headers=True) method. " - "See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html" - ), - ) from e - - headers.update(request_wrapper.headers) - return headers, body - - def _sign_with_manual_credentials( - self, - headers: dict, - optional_params: dict, - request_data: dict, - api_base: str, - ) -> Tuple[dict, None]: - """ - Sign request using manual OCI credentials. - - Args: - headers: Request headers to be signed - optional_params: Optional parameters including OCI credentials - request_data: The request body dict to be sent in HTTP request - api_base: The complete URL for the HTTP request - - Returns: - Tuple of (signed_headers, None) - - Raises: - Exception: If required credentials are missing - ImportError: If cryptography package is not installed - """ - oci_region = optional_params.get("oci_region", "us-ashburn-1") - api_base = ( - api_base - or litellm.api_base - or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" - ) - oci_user = optional_params.get("oci_user") - oci_fingerprint = optional_params.get("oci_fingerprint") - oci_tenancy = optional_params.get("oci_tenancy") - oci_key = optional_params.get("oci_key") - oci_key_file = optional_params.get("oci_key_file") - - if ( - not oci_user - or not oci_fingerprint - or not oci_tenancy - or not (oci_key or oci_key_file) - ): - raise Exception( - "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, " - "and at least one of oci_key or oci_key_file." - ) - - method = str(optional_params.get("method", "POST")).upper() - body = json.dumps(request_data).encode("utf-8") - parsed = urlparse(api_base) - path = parsed.path or "/" - host = parsed.netloc - - date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - content_type = headers.get("content-type", "application/json") - content_length = str(len(body)) - x_content_sha256 = sha256_base64(body) - - headers_to_sign = { - "date": date, - "host": host, - "content-type": content_type, - "content-length": content_length, - "x-content-sha256": x_content_sha256, - } - - signed_headers = [ - "date", - "(request-target)", - "host", - "content-length", - "content-type", - "x-content-sha256", - ] - signing_string = build_signature_string( - method, path, headers_to_sign, signed_headers - ) - - try: - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.asymmetric import padding - except ImportError as e: - raise ImportError( - "cryptography package is required for OCI authentication. " - "Please install it with: pip install cryptography" - ) from e - - # Handle oci_key - it should be a string (PEM content) - oci_key_content = None - if oci_key: - if isinstance(oci_key, str): - oci_key_content = oci_key - # Fix common issues with PEM content - # Replace escaped newlines with actual newlines - oci_key_content = oci_key_content.replace("\\n", "\n") - # Ensure proper line endings - if "\r\n" in oci_key_content: - oci_key_content = oci_key_content.replace("\r\n", "\n") - else: - raise OCIError( - status_code=400, - message=f"oci_key must be a string containing the PEM private key content. " - f"Got type: {type(oci_key).__name__}", - ) - - private_key = ( - load_private_key_from_str(oci_key_content) - if oci_key_content - else load_private_key_from_file(oci_key_file) if oci_key_file else None - ) - - if private_key is None: - raise OCIError( - status_code=400, - message="Private key is required for OCI authentication. Please provide either oci_key or oci_key_file.", - ) - - signature = private_key.sign( - signing_string.encode("utf-8"), - padding.PKCS1v15(), - hashes.SHA256(), - ) - signature_b64 = base64.b64encode(signature).decode() - - key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}" - - authorization = ( - 'Signature version="1",' - f'keyId="{key_id}",' - 'algorithm="rsa-sha256",' - f'headers="{" ".join(signed_headers)}",' - f'signature="{signature_b64}"' - ) - - headers.update( - { - "authorization": authorization, - "date": date, - "host": host, - "content-type": content_type, - "content-length": content_length, - "x-content-sha256": x_content_sha256, - } - ) - - return headers, None - def sign_request( self, headers: dict, @@ -510,61 +346,16 @@ def sign_request( model: Optional[str] = None, stream: Optional[bool] = None, fake_stream: Optional[bool] = None, - ) -> Tuple[dict, Optional[bytes]]: - """ - Sign the OCI request by adding authentication headers. - - Supports two signing modes: - 1. OCI SDK Signer: Use an oci_signer object to sign the request - 2. Manual Signing: Use OCI credentials to manually sign the request - - Args: - headers: Request headers to be signed - optional_params: Optional parameters including auth credentials or oci_signer - request_data: The request body dict to be sent in HTTP request - api_base: The complete URL for the HTTP request - api_key: Optional API key (not used for OCI) - model: Optional model name - stream: Optional streaming flag - fake_stream: Optional fake streaming flag - - Returns: - Tuple of (signed_headers, encoded_body): - - If oci_signer is provided: Returns (headers, body) where body is the encoded JSON - - If manual credentials are provided: Returns (headers, None) as body is not returned - for the manual signing path - - Raises: - OCIError: If signing fails with oci_signer - Exception: If required credentials are missing - ImportError: If cryptography package is not installed (manual signing only) - - Example: - >>> from oci.signer import Signer - >>> signer = Signer( - ... tenancy="ocid1.tenancy.oc1..", - ... user="ocid1.user.oc1..", - ... fingerprint="xx:xx:xx", - ... private_key_file_location="~/.oci/key.pem" - ... ) - >>> headers, body = config.sign_request( - ... headers={}, - ... optional_params={"oci_signer": signer}, - ... request_data={"message": "Hello"}, - ... api_base="https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/..." - ... ) - """ - oci_signer = optional_params.get("oci_signer") - - # If a signer is provided, use it for request signing - if oci_signer is not None: - return self._sign_with_oci_signer( - headers, optional_params, request_data, api_base - ) - - # Standard manual credential signing - return self._sign_with_manual_credentials( - headers, optional_params, request_data, api_base + ) -> Tuple[dict, bytes]: + return sign_oci_request( + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, ) def validate_environment( @@ -577,80 +368,35 @@ def validate_environment( api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - """ - Validate the OCI environment and credentials. - - Supports two authentication modes: - 1. OCI SDK Signer: Pass an oci_signer object (e.g., oci.signer.Signer) - 2. Manual Credentials: Pass oci_user, oci_fingerprint, oci_tenancy, and oci_key/oci_key_file - - Args: - headers: Request headers to populate - model: Model name - messages: List of chat messages - optional_params: Optional parameters including authentication credentials - litellm_params: LiteLLM parameters - api_key: Optional API key (not used for OCI) - api_base: Optional API base URL - - Returns: - Updated headers dict - - Raises: - Exception: If required parameters are missing or invalid - """ - oci_signer = optional_params.get("oci_signer") - oci_region = optional_params.get("oci_region", "us-ashburn-1") - - # Determine api_base - api_base = ( - api_base - or litellm.api_base - or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" - ) - - if not api_base: - raise Exception( - "Either `api_base` must be provided or `litellm.api_base` must be set. " - "Alternatively, you can set the `oci_region` optional parameter to use the default OCI region." - ) - - # Validate credentials only if signer is not provided - if oci_signer is None: - oci_user = optional_params.get("oci_user") - oci_fingerprint = optional_params.get("oci_fingerprint") - oci_tenancy = optional_params.get("oci_tenancy") - oci_key = optional_params.get("oci_key") - oci_key_file = optional_params.get("oci_key_file") - oci_compartment_id = optional_params.get("oci_compartment_id") - - if ( - not oci_user - or not oci_fingerprint - or not oci_tenancy - or not (oci_key or oci_key_file) - or not oci_compartment_id - ): - raise Exception( - "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " - "and at least one of oci_key or oci_key_file. " - "Alternatively, provide an oci_signer object from the OCI SDK." - ) - - # Common header setup - headers.update( - { - "content-type": "application/json", - "user-agent": f"litellm/{version}", - } - ) - if not messages: - raise Exception( - "kwarg `messages` must be an array of messages that follow the openai chat standard" + raise OCIError( + status_code=400, + message="kwarg `messages` must be an array of messages that follow the openai chat standard", ) - - return headers + if optional_params.get("oci_signer") is None: + creds = resolve_oci_credentials(optional_params) + missing = [ + k + for k in ( + "oci_user", + "oci_fingerprint", + "oci_tenancy", + "oci_compartment_id", + ) + if not creds.get(k) + ] + if missing or not (creds.get("oci_key") or creds.get("oci_key_file")): + raise OCIError( + status_code=401, + message=( + "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, " + "oci_compartment_id and at least one of oci_key or oci_key_file. " + "These can be supplied via optional_params or via OCI_USER, OCI_FINGERPRINT, " + "OCI_TENANCY, OCI_COMPARTMENT_ID, OCI_KEY_FILE environment variables. " + "Alternatively, provide an oci_signer object from the OCI SDK." + ), + ) + return validate_oci_environment(headers, optional_params, api_key) def get_complete_url( self, @@ -661,43 +407,63 @@ def get_complete_url( litellm_params: dict, stream: Optional[bool] = None, ) -> str: - oci_region = optional_params.get("oci_region", "us-ashburn-1") - return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/chat" + base = get_oci_base_url(optional_params, api_base or litellm.api_base) + return f"{base}/{OCI_API_VERSION}/actions/chat" + + def _get_optional_params( + self, vendor: OCIVendors, optional_params: dict, model: str = "" + ) -> Dict: + param_map = ( + self.openai_to_oci_cohere_param_map + if vendor == OCIVendors.COHERE + else self.openai_to_oci_generic_param_map + ) + selected_params: Dict = {} + + # OpenAI reasoning models on OCI (e.g. GPT-5 family) reject "maxTokens" + # and require "maxCompletionTokens" per OCI's /20231130/Chat schema. + # Driven by the supports_reasoning flag in the model catalog. Cohere's + # endpoint uses "maxTokens" regardless, so the override is GENERIC-only. + max_tokens_key = ( + "maxCompletionTokens" + if vendor != OCIVendors.COHERE + and model + and _model_uses_max_completion_tokens(model) + else "maxTokens" + ) - def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict: - selected_params = {} - if vendor == OCIVendors.COHERE: - open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map - # remove tool_choice from the map - open_ai_to_oci_param_map.pop("tool_choice") - # Add default values for Cohere API - selected_params = { - "maxTokens": 600, - "temperature": 1, - "topK": 0, - "topP": 0.75, - "frequencyPenalty": 0, - } - else: - open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map - - # Map OpenAI params to OCI params - for openai_key, oci_key in open_ai_to_oci_param_map.items(): - if oci_key and openai_key in optional_params: - selected_params[oci_key] = optional_params[openai_key] # type: ignore[index] - - # Also check for already-mapped OCI params (for backward compatibility) - for oci_value in open_ai_to_oci_param_map.values(): - if ( - oci_value - and oci_value in optional_params - and oci_value not in selected_params - ): - selected_params[oci_value] = optional_params[oci_value] # type: ignore[index] + # ``map_openai_params`` runs before ``transform_request`` (and thus + # before this helper), so by the time we see ``optional_params`` the + # OpenAI keys have already been translated to their OCI aliases. + # We still accept the original OpenAI key as a fallback for callers + # that build ``optional_params`` directly, with OpenAI keys winning + # over OCI aliases when both happen to be present. The first OpenAI + # key reaching a given OCI target wins, so ``max_tokens`` / + # ``max_completion_tokens`` (both → ``maxTokens``) don't double-write. + for openai_key, oci_alias in param_map.items(): + if not oci_alias: + continue + target = max_tokens_key if oci_alias == "maxTokens" else oci_alias + if target in selected_params: + continue + if openai_key in optional_params: + selected_params[target] = optional_params[openai_key] # type: ignore[index] + elif oci_alias in optional_params: + selected_params[target] = optional_params[oci_alias] # type: ignore[index] + + # OCI expects uppercase reasoning levels (LOW/MEDIUM/HIGH/NONE); OpenAI + # clients send lowercase. OpenAI's "disable" maps to OCI's "NONE". + if "reasoningEffort" in selected_params: + effort = selected_params["reasoningEffort"] + if isinstance(effort, str): + normalized = effort.upper() + if normalized == "DISABLE": + normalized = "NONE" + selected_params["reasoningEffort"] = normalized if "tools" in selected_params: if vendor == OCIVendors.COHERE: - selected_params["tools"] = self.adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment] + selected_params["tools"] = adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment] selected_params["tools"] # type: ignore[arg-type] ) else: @@ -705,145 +471,14 @@ def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dic selected_params["tools"], vendor # type: ignore[arg-type] ) - # Transform response_format type to OCI uppercase format - if "responseFormat" in selected_params: - rf = selected_params["responseFormat"] - if isinstance(rf, dict) and "type" in rf: - rf_payload = dict(rf) - selected_params["responseFormat"] = rf_payload - - response_type = rf_payload["type"] - schema_payload: Optional[Any] = None - - if "json_schema" in rf_payload: - raw_schema_payload = rf_payload.pop("json_schema") - if isinstance(raw_schema_payload, dict): - schema_payload = dict(raw_schema_payload) - else: - schema_payload = raw_schema_payload - - if schema_payload is not None: - rf_payload["jsonSchema"] = schema_payload - - if vendor == OCIVendors.COHERE: - # Cohere expects lower-case type values - rf_payload["type"] = response_type - else: - format_type = response_type.upper() - if format_type == "JSON": - format_type = "JSON_OBJECT" - rf_payload["type"] = format_type - - return selected_params - - def adapt_messages_to_cohere_standard( - self, messages: List[AllMessageValues] - ) -> List[CohereMessage]: - """Build chat history for Cohere models.""" - chat_history = [] - for msg in messages[:-1]: # All messages except the last one - role = msg.get("role") - content = msg.get("content") - - if isinstance(content, list): - # Extract text from content array - text_content = "" - for content_item in content: - if ( - isinstance(content_item, dict) - and content_item.get("type") == "text" - ): - text_content += content_item.get("text", "") - content = text_content - - # Ensure content is a string - if not isinstance(content, str): - content = str(content) if content is not None else "" - - # Handle tool calls - tool_calls: Optional[List[CohereToolCall]] = None - if role == "assistant" and "tool_calls" in msg and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item] - tool_calls = [] - for tool_call in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item] - # Parse arguments if they're a JSON string - raw_arguments: Any = tool_call.get("function", {}).get( - "arguments", {} - ) - if isinstance(raw_arguments, str): - try: - arguments: Dict[str, Any] = json.loads(raw_arguments) - except json.JSONDecodeError: - arguments = {} - else: - arguments = raw_arguments - - tool_calls.append( - CohereToolCall( - name=str(tool_call.get("function", {}).get("name", "")), - parameters=arguments, - ) - ) - - if role == "user": - chat_history.append(CohereMessage(role="USER", message=content)) - elif role == "assistant": - chat_history.append( - CohereMessage(role="CHATBOT", message=content, toolCalls=tool_calls) - ) - elif role == "tool": - # Tool messages need special handling - chat_history.append( - CohereMessage( - role="TOOL", - message=content, - toolCalls=None, # Tool messages don't have tool calls - ) - ) + # Normalise tool_choice to OCI's flat uppercase dict form + # ({"type": "AUTO"|"NONE"|"REQUIRED"} or {"type": "FUNCTION", "name": ""}). + # OCI rejects both the OpenAI string and the nested OpenAI dict shape. + _normalize_tool_choice(selected_params) - return chat_history - - def adapt_tool_definitions_to_cohere_standard( - self, tools: List[Dict[str, Any]] - ) -> List[CohereTool]: - """Adapt tool definitions to Cohere format.""" - cohere_tools = [] - for tool in tools: - function_def = tool.get("function", {}) - parameters = function_def.get("parameters", {}).get("properties", {}) - required = function_def.get("parameters", {}).get("required", []) - - parameter_definitions = {} - for param_name, param_schema in parameters.items(): - parameter_definitions[param_name] = CohereParameterDefinition( - description=param_schema.get("description", ""), - type=param_schema.get("type", "string"), - isRequired=param_name in required, - ) + _normalize_response_format(selected_params, vendor) - cohere_tools.append( - CohereTool( - name=function_def.get("name", ""), - description=function_def.get("description", ""), - parameterDefinitions=parameter_definitions, - ) - ) - - return cohere_tools - - def _extract_text_content(self, content: Any) -> str: - """Extract text content from message content.""" - if isinstance(content, str): - return content - elif isinstance(content, list): - text_content = "" - for content_item in content: - if ( - isinstance(content_item, dict) - and content_item.get("type") == "text" - ): - text_content += content_item.get("text", "") - return text_content - return str(content) + return selected_params def transform_request( self, @@ -853,186 +488,78 @@ def transform_request( litellm_params: dict, headers: dict, ) -> dict: - oci_compartment_id = optional_params.get("oci_compartment_id", None) + creds = resolve_oci_credentials(optional_params) + oci_compartment_id = creds["oci_compartment_id"] if not oci_compartment_id: - raise Exception("kwarg `oci_compartment_id` is required for OCI requests") + raise OCIError( + status_code=400, + message=( + "oci_compartment_id is required for OCI chat requests. " + "Pass it as optional_params or set the OCI_COMPARTMENT_ID env var." + ), + ) vendor = get_vendor_from_model(model) oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND") if oci_serving_mode not in ["ON_DEMAND", "DEDICATED"]: - raise Exception( - "kwarg `oci_serving_mode` must be either 'ON_DEMAND' or 'DEDICATED'" + raise OCIError( + status_code=400, + message="kwarg `oci_serving_mode` must be either 'ON_DEMAND' or 'DEDICATED'", ) if oci_serving_mode == "DEDICATED": - oci_endpoint_id = optional_params.get("oci_endpoint_id", model) - servingMode = OCIServingMode( + serving_mode = OCIServingMode( servingType="DEDICATED", - endpointId=oci_endpoint_id, + endpointId=optional_params.get("oci_endpoint_id", model), ) else: - servingMode = OCIServingMode( - servingType="ON_DEMAND", - modelId=model, - ) + serving_mode = OCIServingMode(servingType="ON_DEMAND", modelId=model) - # Build request based on vendor type if vendor == OCIVendors.COHERE: - # For Cohere, we need to use the specific Cohere format - # Extract the last user message as the main message - user_messages = [msg for msg in messages if msg.get("role") == "user"] + user_messages = [m for m in messages if m.get("role") == "user"] if not user_messages: - raise Exception("No user message found for Cohere model") + raise OCIError( + status_code=400, + message="No user message found — Cohere models require at least one user message", + ) - # Extract system messages into preambleOverride - system_messages = [msg for msg in messages if msg.get("role") == "system"] + system_messages = [m for m in messages if m.get("role") == "system"] preamble_override = None if system_messages: preamble = "\n".join( - self._extract_text_content(msg["content"]) - for msg in system_messages + _extract_text_content(m["content"]) for m in system_messages ) if preamble: preamble_override = preamble - # Create Cohere-specific chat request - optional_cohere_params = self._get_optional_params( - OCIVendors.COHERE, optional_params - ) chat_request = CohereChatRequest( apiFormat="COHERE", - message=self._extract_text_content(user_messages[-1]["content"]), - chatHistory=self.adapt_messages_to_cohere_standard(messages), + message=_extract_text_content(user_messages[-1]["content"]), + chatHistory=adapt_messages_to_cohere_standard( + [m for m in messages if m.get("role") != "system"] + ), preambleOverride=preamble_override, - **optional_cohere_params, + **self._get_optional_params(OCIVendors.COHERE, optional_params, model), ) - data = OCICompletionPayload( compartmentId=oci_compartment_id, - servingMode=servingMode, + servingMode=serving_mode, chatRequest=chat_request, ) else: - # Use generic format for other vendors data = OCICompletionPayload( compartmentId=oci_compartment_id, - servingMode=servingMode, + servingMode=serving_mode, chatRequest=OCIChatRequestPayload( apiFormat=vendor.value, messages=adapt_messages_to_generic_oci_standard(messages), - **self._get_optional_params(vendor, optional_params), + **self._get_optional_params(vendor, optional_params, model), ), ) return data.model_dump(exclude_none=True) - def _handle_cohere_response( - self, json_response: dict, model: str, model_response: ModelResponse - ) -> ModelResponse: - """Handle Cohere-specific response format.""" - cohere_response = CohereChatResult(**json_response) - # Cohere response format (uses camelCase) - model_id = model - - # Set basic response info - model_response.model = model_id - model_response.created = int(datetime.datetime.now().timestamp()) - - # Extract the response text - response_text = cohere_response.chatResponse.text - oci_finish_reason = cohere_response.chatResponse.finishReason - - # Map finish reason - if oci_finish_reason == "COMPLETE": - finish_reason = "stop" - elif oci_finish_reason == "MAX_TOKENS": - finish_reason = "length" - else: - finish_reason = "stop" - - # Handle tool calls - tool_calls: Optional[List[Dict[str, Any]]] = None - if cohere_response.chatResponse.toolCalls: - tool_calls = [] - for tool_call in cohere_response.chatResponse.toolCalls: - tool_calls.append( - { - "id": f"call_{len(tool_calls)}", # Generate a simple ID - "type": "function", - "function": { - "name": tool_call.name, - "arguments": json.dumps(tool_call.parameters), - }, - } - ) - - # Create choice - from litellm.types.utils import Choices - - choice = Choices( - index=0, - message={ - "role": "assistant", - "content": response_text, - "tool_calls": tool_calls, - }, - finish_reason=finish_reason, - ) - model_response.choices = [choice] - - # Extract usage info - usage_info = cohere_response.chatResponse.usage - from litellm.types.utils import Usage - - model_response.usage = Usage( # type: ignore[attr-defined] - prompt_tokens=usage_info.promptTokens, # type: ignore[union-attr] - completion_tokens=usage_info.completionTokens, # type: ignore[union-attr] - total_tokens=usage_info.totalTokens, # type: ignore[union-attr] - ) - - return model_response - - def _handle_generic_response( - self, - json: dict, - model: str, - model_response: ModelResponse, - raw_response: httpx.Response, - ) -> ModelResponse: - """Handle generic OCI response format.""" - try: - completion_response = OCICompletionResponse(**json) - except TypeError as e: - raise OCIError( - message=f"Response cannot be casted to OCICompletionResponse: {str(e)}", - status_code=raw_response.status_code, - ) - - iso_str = completion_response.chatResponse.timeCreated - dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00")) - model_response.created = int(dt.timestamp()) - - model_response.model = completion_response.modelId - - message = model_response.choices[0].message # type: ignore - response_message = completion_response.chatResponse.choices[0].message - if response_message.content and response_message.content[0].type == "TEXT": - message.content = response_message.content[0].text - if response_message.toolCalls: - message.tool_calls = adapt_tools_to_openai_standard( - response_message.toolCalls - ) - - usage = Usage( - prompt_tokens=completion_response.chatResponse.usage.promptTokens, - completion_tokens=completion_response.chatResponse.usage.completionTokens, - total_tokens=completion_response.chatResponse.usage.totalTokens, - ) - model_response.usage = usage # type: ignore - - return model_response - def transform_response( self, model: str, @@ -1047,34 +574,31 @@ def transform_response( api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: - json = raw_response.json() # noqa: F811 + response_json = raw_response.json() - error = json.get("error") - - if error is not None: + if not isinstance(response_json, dict): raise OCIError( - message=str(json["error"]), + message="Invalid response format from OCI", status_code=raw_response.status_code, ) - if not isinstance(json, dict): + if response_json.get("error") is not None: raise OCIError( - message="Invalid response format from OCI", + message=str(response_json["error"]), status_code=raw_response.status_code, ) vendor = get_vendor_from_model(model) - - # Handle response based on vendor type if vendor == OCIVendors.COHERE: - model_response = self._handle_cohere_response(json, model, model_response) + model_response = handle_cohere_response( + response_json, model, model_response, raw_response + ) else: - model_response = self._handle_generic_response( - json, model, model_response, raw_response + model_response = handle_generic_response( + response_json, model, model_response, raw_response ) model_response._hidden_params["additional_headers"] = raw_response.headers - return model_response @track_llm_api_timing() @@ -1091,8 +615,6 @@ def get_sync_custom_stream_wrapper( json_mode: Optional[bool] = None, signed_json_body: Optional[bytes] = None, ) -> "OCIStreamWrapper": - if "stream" in data: - del data["stream"] if client is None or isinstance(client, AsyncHTTPHandler): client = _get_httpx_client(params={}) @@ -1100,7 +622,11 @@ def get_sync_custom_stream_wrapper( response = client.post( api_base, headers=headers, - data=json.dumps(data), + data=( + signed_json_body + if signed_json_body is not None + else json.dumps(data) + ), stream=True, logging_obj=logging_obj, timeout=STREAMING_TIMEOUT, @@ -1111,15 +637,12 @@ def get_sync_custom_stream_wrapper( if response.status_code != 200: raise OCIError(status_code=response.status_code, message=response.text) - completion_stream = response.iter_text() - - streaming_response = OCIStreamWrapper( - completion_stream=completion_stream, + return OCIStreamWrapper( + completion_stream=_iter_sse_events(response.iter_text()), model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) - return streaming_response @track_llm_api_timing() async def get_async_custom_stream_wrapper( @@ -1135,17 +658,18 @@ async def get_async_custom_stream_wrapper( json_mode: Optional[bool] = None, signed_json_body: Optional[bytes] = None, ) -> "OCIStreamWrapper": - if "stream" in data: - del data["stream"] - if client is None or isinstance(client, HTTPHandler): - client = get_async_httpx_client(llm_provider=LlmProviders.BYTEZ, params={}) + client = get_async_httpx_client(llm_provider=LlmProviders.OCI, params={}) try: response = await client.post( api_base, headers=headers, - data=json.dumps(data), + data=( + signed_json_body + if signed_json_body is not None + else json.dumps(data) + ), stream=True, logging_obj=logging_obj, timeout=STREAMING_TIMEOUT, @@ -1156,22 +680,12 @@ async def get_async_custom_stream_wrapper( if response.status_code != 200: raise OCIError(status_code=response.status_code, message=response.text) - completion_stream = response.aiter_text() - - async def split_chunks(completion_stream: AsyncIterator[str]): - async for item in completion_stream: - for chunk in item.split("\n\n"): - if not chunk: - continue - yield chunk.strip() - - streaming_response = OCIStreamWrapper( - completion_stream=split_chunks(completion_stream), + return OCIStreamWrapper( + completion_stream=_aiter_sse_events(response.aiter_text()), model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) - return streaming_response def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] @@ -1179,332 +693,61 @@ def get_error_class( return OCIError(status_code=status_code, message=error_message) -open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = { - "system": "SYSTEM", - "user": "USER", - "assistant": "ASSISTANT", - "tool": "TOOL", -} - - -def adapt_messages_to_generic_oci_standard_content_message( - role: str, content: Union[str, list] -) -> OCIMessage: - new_content: List[OCIContentPartUnion] = [] - if isinstance(content, str): - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=[OCITextContentPart(text=content)], - toolCalls=None, - toolCallId=None, - ) - - # content is a list of content items: - # [ - # {"type": "text", "text": "Hello"}, - # {"type": "image_url", "image_url": "https://example.com/image.png"} - # ] - for content_item in content: - if not isinstance(content_item, dict): - raise Exception("Each content item must be a dictionary") - - type = content_item.get("type") - if not isinstance(type, str): - raise Exception("Prop `type` is not a string") - - if type not in ["text", "image_url"]: - raise Exception(f"Prop `{type}` is not supported") - - if type == "text": - text = content_item.get("text") - if not isinstance(text, str): - raise Exception("Prop `text` is not a string") - new_content.append(OCITextContentPart(text=text)) - - elif type == "image_url": - image_url = content_item.get("image_url") - # Handle both OpenAI format (object with url) and string format - if isinstance(image_url, dict): - image_url = image_url.get("url") - if not isinstance(image_url, str): - raise Exception( - "Prop `image_url` must be a string or an object with a `url` property" - ) - new_content.append(OCIImageContentPart(imageUrl=OCIImageUrl(url=image_url))) - - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=new_content, - toolCalls=None, - toolCallId=None, - ) - - -def adapt_messages_to_generic_oci_standard_tool_call( - role: str, tool_calls: list -) -> OCIMessage: - tool_calls_formated = [] - for tool_call in tool_calls: - if not isinstance(tool_call, dict): - raise Exception("Each tool call must be a dictionary") - - if tool_call.get("type") != "function": - raise Exception("OCI only supports function tools") - - tool_call_id = tool_call.get("id") - if not isinstance(tool_call_id, str): - raise Exception("Prop `id` is not a string") - - tool_function = tool_call.get("function") - if not isinstance(tool_function, dict): - raise Exception("Prop `function` is not a dictionary") - - function_name = tool_function.get("name") - if not isinstance(function_name, str): - raise Exception("Prop `name` is not a string") - - arguments = tool_call["function"].get("arguments", "{}") - if not isinstance(arguments, str): - raise Exception("Prop `arguments` is not a string") - - # tool_calls_formated.append(OCIToolCall( - # id=tool_call_id, - # type="FUNCTION", - # function=OCIFunction( - # name=function_name, - # arguments=arguments - # ) - # )) - - tool_calls_formated.append( - OCIToolCall( - id=tool_call_id, - type="FUNCTION", - name=function_name, - arguments=arguments, - ) - ) - - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=None, - toolCalls=tool_calls_formated, - toolCallId=None, - ) - - -def adapt_messages_to_generic_oci_standard_tool_response( - role: str, tool_call_id: str, content: str -) -> OCIMessage: - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=[OCITextContentPart(text=content)], - toolCalls=None, - toolCallId=tool_call_id, - ) - - -def adapt_messages_to_generic_oci_standard( - messages: List[AllMessageValues], -) -> List[OCIMessage]: - new_messages = [] - for message in messages: - role = message["role"] - content = message.get("content") - tool_calls = message.get("tool_calls") - tool_call_id = message.get("tool_call_id") - - if role == "assistant" and tool_calls is not None: - if not isinstance(tool_calls, list): - raise Exception("Prop `tool_calls` must be a list of tool calls") - new_messages.append( - adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) - ) - - elif role in ["system", "user", "assistant"] and content is not None: - if not isinstance(content, (str, list)): - raise Exception( - "Prop `content` must be a string or a list of content items" - ) - new_messages.append( - adapt_messages_to_generic_oci_standard_content_message(role, content) - ) - - elif role == "tool": - if not isinstance(tool_call_id, str): - raise Exception("Prop `tool_call_id` is required and must be a string") - if not isinstance(content, str): - raise Exception("Prop `content` is not a string") - new_messages.append( - adapt_messages_to_generic_oci_standard_tool_response( - role, tool_call_id, content - ) - ) - - return new_messages - - -def adapt_tool_definition_to_oci_standard(tools: List[Dict], vendor: OCIVendors): - new_tools = [] - for tool in tools: - if tool["type"] != "function": - raise Exception("OCI only supports function tools") - - tool_function = tool.get("function") - if not isinstance(tool_function, dict): - raise Exception("Prop `function` is not a dictionary") - - new_tool = OCIToolDefinition( - type="FUNCTION", - name=tool_function.get("name"), - description=tool_function.get("description", ""), - parameters=tool_function.get("parameters", {}), - ) - new_tools.append(new_tool) - - return new_tools - - -def adapt_tools_to_openai_standard( - tools: List[OCIToolCall], -) -> List[ChatCompletionMessageToolCall]: - new_tools = [] - for tool in tools: - new_tool = ChatCompletionMessageToolCall( - id=tool.id, - type="function", - function={ - "name": tool.name, - "arguments": tool.arguments, - }, - ) - new_tools.append(new_tool) - return new_tools - - class OCIStreamWrapper(CustomStreamWrapper): - """ - Custom stream wrapper for OCI responses. - This class is used to handle streaming responses from OCI's API. - """ + """Custom stream wrapper that dispatches OCI SSE chunks to the correct handler.""" - def __init__( - self, - **kwargs: Any, - ): + def __init__(self, **kwargs: Any): super().__init__(**kwargs) - - def chunk_creator(self, chunk: Any): + # Tracks whether any prior Cohere chunk in this stream has emitted + # tool calls. The Cohere handler uses this to decide whether the + # terminal consolidation chunk's tool calls are duplicates (suppress) + # or the only copy of the tool calls (pass through). + self._cohere_tool_calls_emitted = False + # Analogous flag for text content. Lets the Cohere handler distinguish + # the common case (prior deltas already streamed the text, so the + # terminal chunk's text is a duplicate to suppress) from the degenerate + # single-event case (terminal chunk carries the only copy of the text). + self._cohere_text_emitted = False + + def chunk_creator(self, chunk: Any) -> ModelResponseStream: if not isinstance(chunk, str): raise ValueError(f"Chunk is not a string: {chunk}") if not chunk.startswith("data:"): raise ValueError(f"Chunk does not start with 'data:': {chunk}") - dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON - - # Check if this is a Cohere stream chunk - if "apiFormat" in dict_chunk and dict_chunk.get("apiFormat") == "COHERE": - return self._handle_cohere_stream_chunk(dict_chunk) - else: - return self._handle_generic_stream_chunk(dict_chunk) - - def _handle_cohere_stream_chunk(self, dict_chunk: dict): - """Handle Cohere-specific streaming chunks.""" try: - typed_chunk = CohereStreamChunk(**dict_chunk) - except TypeError as e: - raise ValueError(f"Chunk cannot be casted to CohereStreamChunk: {str(e)}") - - if typed_chunk.index is None: - typed_chunk.index = 0 - - # Extract text content - text = typed_chunk.text or "" - - # Map finish reason to standard format - finish_reason = typed_chunk.finishReason - if finish_reason == "COMPLETE": - finish_reason = "stop" - elif finish_reason == "MAX_TOKENS": - finish_reason = "length" - elif finish_reason is None: - finish_reason = None - else: - finish_reason = "stop" - - # For Cohere, we don't have tool calls in the streaming format - tool_calls = None - - return ModelResponseStream( - choices=[ - StreamingChoices( - index=typed_chunk.index if typed_chunk.index else 0, - delta=Delta( - content=text, - tool_calls=tool_calls, - provider_specific_fields=None, - thinking_blocks=None, - reasoning_content=None, - ), - finish_reason=finish_reason, - ) - ] - ) - - def _handle_generic_stream_chunk(self, dict_chunk: dict): - """Handle generic OCI streaming chunks.""" - # Fix missing required fields in tool calls before Pydantic validation - # OCI streams tool calls progressively, so early chunks may be missing required fields - if dict_chunk.get("message") and dict_chunk["message"].get("toolCalls"): - for tool_call in dict_chunk["message"]["toolCalls"]: - if "arguments" not in tool_call: - tool_call["arguments"] = "" - if "id" not in tool_call: - tool_call["id"] = "" - if "name" not in tool_call: - tool_call["name"] = "" + dict_chunk = json.loads(chunk[5:]) + except json.JSONDecodeError as e: + raise OCIError( + status_code=500, + message=f"Chunk cannot be parsed as JSON: {str(e)}", + ) - try: - typed_chunk = OCIStreamChunk(**dict_chunk) - except TypeError as e: - raise ValueError(f"Chunk cannot be casted to OCIStreamChunk: {str(e)}") - - if typed_chunk.index is None: - typed_chunk.index = 0 - - text = "" - if typed_chunk.message and typed_chunk.message.content: - for item in typed_chunk.message.content: - if isinstance(item, OCITextContentPart): - text += item.text - elif isinstance(item, OCIImageContentPart): - raise ValueError( - "OCI does not support image content in streaming responses" - ) - else: - raise ValueError( - f"Unsupported content type in OCI response: {item.type}" - ) - - tool_calls = None - if typed_chunk.message and typed_chunk.message.toolCalls: - tool_calls = adapt_tools_to_openai_standard(typed_chunk.message.toolCalls) - - return ModelResponseStream( - choices=[ - StreamingChoices( - index=typed_chunk.index if typed_chunk.index else 0, - delta=Delta( - content=text, - tool_calls=( - [tool.model_dump() for tool in tool_calls] - if tool_calls - else None - ), - provider_specific_fields=None, # OCI does not have provider specific fields in the response - thinking_blocks=None, # OCI does not have thinking blocks in the response - reasoning_content=None, # OCI does not have reasoning content in the response - ), - finish_reason=typed_chunk.finishReason, - ) - ] - ) + if dict_chunk.get("apiFormat") == "COHERE": + result = handle_cohere_stream_chunk( + dict_chunk, + prior_tool_calls_emitted=self._cohere_tool_calls_emitted, + prior_text_emitted=self._cohere_text_emitted, + ) + if not self._cohere_tool_calls_emitted: + for choice in result.choices: + if getattr(choice.delta, "tool_calls", None) is not None: + self._cohere_tool_calls_emitted = True + break + if not self._cohere_text_emitted: + for choice in result.choices: + if getattr(choice.delta, "content", None): + self._cohere_text_emitted = True + break + return result + return handle_generic_stream_chunk(dict_chunk) + + +__all__ = [ + "OCIChatConfig", + "OCIStreamWrapper", + "OCIRequestWrapper", + "OCI_API_VERSION", + "STREAMING_TIMEOUT", + "get_vendor_from_model", + "version", +] diff --git a/litellm/llms/oci/common_utils.py b/litellm/llms/oci/common_utils.py index 661a6c89e4b8..8785b1548a52 100644 --- a/litellm/llms/oci/common_utils.py +++ b/litellm/llms/oci/common_utils.py @@ -1,9 +1,42 @@ -from typing import Optional +import base64 +import hashlib +import json +import os +import re +from dataclasses import dataclass +from email.utils import formatdate +from typing import Any, Dict, Optional, Protocol, Tuple +from urllib.parse import urlparse import httpx from litellm.llms.base_llm.chat.transformation import BaseLLMException +try: + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding, rsa + + _CRYPTOGRAPHY_AVAILABLE = True +except ImportError: + _CRYPTOGRAPHY_AVAILABLE = False + +try: + from litellm._version import version as _litellm_version +except ImportError: + _litellm_version = "0.0.0" + + +# OCI GenAI REST API version — stable since service launch, unlikely to change +OCI_API_VERSION = "20231130" + + +def _require_cryptography() -> None: + if not _CRYPTOGRAPHY_AVAILABLE: + raise ImportError( + "cryptography package is required for OCI authentication. " + "Please install it with: pip install cryptography" + ) + class OCIError(BaseLLMException): def __init__( @@ -17,3 +50,520 @@ def __init__( message=message, headers=headers, ) + + +# --------------------------------------------------------------------------- +# OCI signing protocol and helpers +# --------------------------------------------------------------------------- + + +class OCISignerProtocol(Protocol): + """ + Protocol for OCI request signers (e.g., oci.signer.Signer). + + Compatible with the OCI Python SDK's Signer class. + See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html + """ + + def do_request_sign( + self, request: Any, *, enforce_content_headers: bool = False + ) -> None: + pass + + +@dataclass +class OCIRequestWrapper: + """ + Wrapper for HTTP requests compatible with OCI signer interface. + + Wraps request data in the format expected by OCI SDK signers, which require + objects with method, url, headers, body, and path_url attributes. + """ + + method: str + url: str + headers: dict + body: bytes + + @property + def path_url(self) -> str: + """Returns the path + query string for OCI signing.""" + parsed = urlparse(self.url) + return parsed.path + ("?" + parsed.query if parsed.query else "") + + +def sha256_base64(data: bytes) -> str: + # SHA-256 is used here to compute the x-content-sha256 header required by the + # OCI HTTP signing specification (RSA-SHA256 request signing), not for password + # or secret hashing. This is the correct and mandated algorithm for this purpose. + # See: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/signingrequests.htm + # + # ``usedforsecurity=False`` declares non-security intent to static analyzers + # (CodeQL ``py/weak-sensitive-data-hashing``) — without it the request body + # gets flagged as "password-like data" via taint tracking. + digest = hashlib.sha256(data, usedforsecurity=False).digest() # noqa: S324 + return base64.b64encode(digest).decode() + + +def build_signature_string( + method: str, path: str, headers: dict, signed_headers: list +) -> str: + lines = [] + for header in signed_headers: + if header == "(request-target)": + value = f"{method.lower()} {path}" + else: + value = headers[header] + lines.append(f"{header}: {value}") + return "\n".join(lines) + + +def load_private_key_from_str(key_str: str) -> Any: + _require_cryptography() + key = serialization.load_pem_private_key( # type: ignore[union-attr] + key_str.encode("utf-8"), + password=None, + ) + if not isinstance(key, rsa.RSAPrivateKey): # type: ignore[union-attr] + raise TypeError( + "The provided private key is not an RSA key, which is required for OCI signing." + ) + return key + + +def load_private_key_from_file(file_path: str) -> Any: + """Loads a private key from a file path.""" + try: + with open(file_path, "r", encoding="utf-8") as f: + key_str = f.read().strip() + except FileNotFoundError: + raise FileNotFoundError(f"Private key file not found: {file_path}") + except OSError as e: + raise OSError(f"Failed to read private key file '{file_path}': {e}") from e + + if not key_str: + raise ValueError(f"Private key file is empty: {file_path}") + + return load_private_key_from_str(key_str) + + +# --------------------------------------------------------------------------- +# Env-var credential resolution +# --------------------------------------------------------------------------- + +_OCI_REGION_ENV = "OCI_REGION" +_OCI_USER_ENV = "OCI_USER" +_OCI_FINGERPRINT_ENV = "OCI_FINGERPRINT" +_OCI_TENANCY_ENV = "OCI_TENANCY" +_OCI_KEY_FILE_ENV = "OCI_KEY_FILE" +_OCI_KEY_ENV = "OCI_KEY" +_OCI_COMPARTMENT_ID_ENV = "OCI_COMPARTMENT_ID" + + +def resolve_oci_credentials(optional_params: dict) -> dict: + """ + Merge OCI credentials from optional_params (explicit, always wins) and + environment variables (fallback). + + Returns a dict with resolved values for: + oci_region, oci_user, oci_fingerprint, oci_tenancy, + oci_key, oci_key_file, oci_compartment_id + """ + return { + "oci_region": optional_params.get("oci_region") + or os.environ.get(_OCI_REGION_ENV) + or "us-ashburn-1", + "oci_user": optional_params.get("oci_user") or os.environ.get(_OCI_USER_ENV), + "oci_fingerprint": optional_params.get("oci_fingerprint") + or os.environ.get(_OCI_FINGERPRINT_ENV), + "oci_tenancy": optional_params.get("oci_tenancy") + or os.environ.get(_OCI_TENANCY_ENV), + "oci_key": optional_params.get("oci_key") or os.environ.get(_OCI_KEY_ENV), + "oci_key_file": optional_params.get("oci_key_file") + or os.environ.get(_OCI_KEY_FILE_ENV), + "oci_compartment_id": optional_params.get("oci_compartment_id") + or os.environ.get(_OCI_COMPARTMENT_ID_ENV), + } + + +_OCI_REGION_RE = re.compile(r"^[a-z][a-z0-9-]{0,30}[a-z0-9]$") +_OCI_ACTION_PATH_RE = re.compile(rf"/{OCI_API_VERSION}/actions/[^/?#]+/?$") + + +def get_oci_base_url(optional_params: dict, api_base: Optional[str] = None) -> str: + """Return the OCI inference base URL, respecting any explicit api_base override. + + If ``api_base`` already ends with a fully-formed OCI action path + (``/{OCI_API_VERSION}/actions/``), that suffix is stripped so callers + can append their own action path without producing a doubled URL. + """ + if api_base: + return _OCI_ACTION_PATH_RE.sub("", api_base).rstrip("/") + creds = resolve_oci_credentials(optional_params) + region = creds["oci_region"] + if not isinstance(region, str) or not _OCI_REGION_RE.match(region): + raise OCIError( + status_code=400, + message=( + f"Invalid OCI region {region!r}: must match " + "^[a-z][a-z0-9-]{0,30}[a-z0-9]$ (e.g. 'us-ashburn-1')." + ), + ) + return f"https://inference.generativeai.{region}.oci.oraclecloud.com" + + +# --------------------------------------------------------------------------- +# Signing implementations (shared by chat, embed, and rerank configs) +# --------------------------------------------------------------------------- + + +def sign_with_oci_signer( + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, +) -> Tuple[dict, bytes]: + """Sign a request using an OCI SDK Signer object passed in optional_params.""" + oci_signer = optional_params.get("oci_signer") + body = json.dumps(request_data).encode("utf-8") + method = str(optional_params.get("method", "POST")).upper() + + if method not in {"POST", "GET", "PUT", "DELETE", "PATCH"}: + raise ValueError(f"Unsupported HTTP method: {method}") + + prepared_headers = {**headers} + prepared_headers.setdefault("content-type", "application/json") + prepared_headers.setdefault("content-length", str(len(body))) + + request_wrapper = OCIRequestWrapper( + method=method, url=api_base, headers=prepared_headers, body=body + ) + + if oci_signer is None: + raise ValueError("oci_signer cannot be None when calling sign_with_oci_signer") + + try: + oci_signer.do_request_sign(request_wrapper, enforce_content_headers=True) + except Exception as e: + raise OCIError( + status_code=500, + message=( + f"Failed to sign request with provided oci_signer: {str(e)}. " + "The signer must implement the OCI SDK Signer interface with a " + "do_request_sign(request, enforce_content_headers=True) method. " + "See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html" + ), + ) from e + + headers.update(request_wrapper.headers) + return headers, body + + +def sign_with_manual_credentials( + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, +) -> Tuple[dict, bytes]: + """Sign a request using manually provided OCI credentials (user/fingerprint/tenancy/key).""" + creds = resolve_oci_credentials(optional_params) + oci_user = creds["oci_user"] + oci_fingerprint = creds["oci_fingerprint"] + oci_tenancy = creds["oci_tenancy"] + oci_key = creds["oci_key"] + oci_key_file = creds["oci_key_file"] + + if ( + not oci_user + or not oci_fingerprint + or not oci_tenancy + or not (oci_key or oci_key_file) + ): + raise OCIError( + status_code=401, + message=( + "Missing required OCI credentials: oci_user, oci_fingerprint, oci_tenancy, " + "and at least one of oci_key or oci_key_file. " + "These can also be supplied via environment variables: " + f"{_OCI_USER_ENV}, {_OCI_FINGERPRINT_ENV}, {_OCI_TENANCY_ENV}, {_OCI_KEY_ENV} (or {_OCI_KEY_FILE_ENV}). " + "Alternatively, provide an oci_signer object from the OCI SDK." + ), + ) + + method = str(optional_params.get("method", "POST")).upper() + body = json.dumps(request_data).encode("utf-8") + parsed = urlparse(api_base) + path = parsed.path or "/" + host = parsed.netloc + + date = formatdate(usegmt=True) + content_type = headers.get("content-type", "application/json") + content_length = str(len(body)) + x_content_sha256 = sha256_base64(body) + + headers_to_sign: Dict[str, str] = { + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + + signed_header_names = [ + "date", + "(request-target)", + "host", + "content-length", + "content-type", + "x-content-sha256", + ] + signing_string = build_signature_string( + method, path, headers_to_sign, signed_header_names + ) + + _require_cryptography() + + # Resolve the private key — prefer inline PEM content over file path + oci_key_content: Optional[str] = None + if oci_key: + if not isinstance(oci_key, str): + raise OCIError( + status_code=400, + message=( + f"oci_key must be a string containing the PEM private key content. " + f"Got type: {type(oci_key).__name__}" + ), + ) + oci_key_content = oci_key.replace("\\n", "\n").replace("\r\n", "\n") + + private_key = ( + load_private_key_from_str(oci_key_content) + if oci_key_content + else load_private_key_from_file(oci_key_file) if oci_key_file else None + ) + + if private_key is None: + raise OCIError( + status_code=400, + message="Private key is required for OCI authentication. Provide either oci_key or oci_key_file.", + ) + + signature = private_key.sign( + signing_string.encode("utf-8"), + padding.PKCS1v15(), # type: ignore[union-attr] + hashes.SHA256(), # type: ignore[union-attr] + ) + signature_b64 = base64.b64encode(signature).decode() + + key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}" + authorization = ( + 'Signature version="1",' + f'keyId="{key_id}",' + 'algorithm="rsa-sha256",' + f'headers="{" ".join(signed_header_names)}",' + f'signature="{signature_b64}"' + ) + + headers.update( + { + "authorization": authorization, + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + ) + return headers, body + + +def sign_oci_request( + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, +) -> Tuple[dict, bytes]: + """ + Route to the appropriate OCI signing method based on what credentials are present. + + If ``oci_signer`` is in optional_params, use the OCI SDK signer object. + Otherwise use manual RSA-SHA256 signing with explicit credentials (which can + also be supplied via OCI_* environment variables). + + Returns: + Tuple of (signed_headers, signed_body_bytes) + """ + if optional_params.get("oci_signer") is not None: + return sign_with_oci_signer(headers, optional_params, request_data, api_base) + return sign_with_manual_credentials( + headers, optional_params, request_data, api_base + ) + + +def validate_oci_environment( + headers: dict, + optional_params: dict, + api_key: Optional[str] = None, +) -> dict: + """ + Populate common OCI request headers (content-type, user-agent). + + Full credential validation is deferred to signing time so that credentials + supplied via environment variables are resolved at call time rather than + at construction time. + """ + headers.setdefault("content-type", "application/json") + headers.setdefault("user-agent", f"litellm/{_litellm_version}") + return headers + + +# --------------------------------------------------------------------------- +# JSON schema utilities for OCI tool definitions +# +# OCI Generative AI does not support JSON Schema extensions ($ref, $defs, +# anyOf). Pydantic v2 emits all three for models with Optional fields or +# nested schemas. The helpers below are ported from the official +# langchain-oracle reference implementation so that tool schemas are always +# valid before they reach the OCI endpoint. +# --------------------------------------------------------------------------- + +# Mapping from JSON Schema type names to Python type names, as expected by +# the OCI Cohere API's CohereParameterDefinition.type field. +OCI_JSON_TO_PYTHON_TYPES: Dict[str, str] = { + "string": "str", + "number": "float", + "boolean": "bool", + "integer": "int", + "array": "List", + "object": "Dict", + "any": "any", +} + + +def resolve_oci_schema_refs(schema: Dict[str, Any]) -> Dict[str, Any]: + """Inline all ``$ref``/``$defs`` references — OCI does not support JSON Schema ``$ref``.""" + defs = schema.get("$defs", {}) + resolving_stack: set = set() + + def _resolve(obj: Any) -> Any: + if isinstance(obj, dict): + if "$ref" in obj: + ref = obj["$ref"] + if ref.startswith("#/$defs/"): + key = ref.split("/")[-1] + if key in resolving_stack: + return {"type": "object"} # break cycles + resolving_stack.add(key) + try: + return _resolve(defs.get(key, obj)) + finally: + resolving_stack.discard(key) + return obj # external $ref — leave unchanged + return {k: _resolve(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_resolve(item) for item in obj] + return obj + + resolved = _resolve(schema) + if isinstance(resolved, dict): + resolved.pop("$defs", None) + return resolved + + +def resolve_oci_schema_anyof(obj: Any) -> Any: + """Resolve Pydantic v2 ``Optional[T]`` → ``anyOf`` patterns. + + Pydantic v2 emits ``{"anyOf": [{"type": "T"}, {"type": "null"}]}`` for + ``Optional[T]``. OCI models don't understand ``anyOf``, so we pick the + first non-null branch and merge top-level metadata into it. + """ + if isinstance(obj, dict): + if "anyOf" in obj and "type" not in obj: + non_null = [ + t + for t in obj["anyOf"] + if not (isinstance(t, dict) and t.get("type") == "null") + ] + if non_null: + resolved = {**obj, **non_null[0]} + resolved.pop("anyOf", None) + return resolve_oci_schema_anyof(resolved) + return {k: resolve_oci_schema_anyof(v) for k, v in obj.items()} + if isinstance(obj, list): + return [resolve_oci_schema_anyof(item) for item in obj] + return obj + + +def sanitize_oci_schema(schema: Any) -> Any: + """Recursively remove OCI-incompatible fields from a JSON schema. + + Strips ``title`` keys, removes ``None``-valued ``default`` entries, + normalises ``type: [T, "null"]`` list types, and ensures arrays carry an + ``items`` definition. + """ + if isinstance(schema, list): + return [sanitize_oci_schema(item) for item in schema] + if not isinstance(schema, dict): + return schema + + sanitized: Dict[str, Any] = {} + for key, value in schema.items(): + if key == "title": + continue + if key == "default" and value is None: + continue + if key == "type": + if value == "any": + sanitized[key] = "object" + continue + if isinstance(value, list): + non_null = [t for t in value if t != "null"] + sanitized[key] = non_null[0] if non_null else "string" + continue + sanitized[key] = sanitize_oci_schema(value) + + if sanitized.get("type") == "array" and "items" not in sanitized: + sanitized["items"] = {"type": "object"} + + required = sanitized.get("required") + properties = sanitized.get("properties") + if "required" in sanitized: + if isinstance(required, list) and isinstance(properties, dict): + sanitized["required"] = [ + f for f in required if isinstance(f, str) and f in properties + ] + elif not isinstance(required, list): + sanitized["required"] = [] + + return sanitized + + +def enrich_cohere_param_description( + description: str, param_schema: Dict[str, Any] +) -> str: + """Embed schema constraints into a Cohere parameter description. + + ``CohereParameterDefinition`` only has ``type``, ``description``, and + ``isRequired``. Rich constraints (``enum``, ``format``, ``minimum``, + ``maximum``, ``pattern``) are appended to the description string so the + model can still see and respect them. + """ + parts = [description] if description else [] + if "enum" in param_schema: + parts.append(f"Allowed values: {param_schema['enum']}") + if "format" in param_schema: + parts.append(f"Format: {param_schema['format']}") + if "minimum" in param_schema or "maximum" in param_schema: + range_parts = [] + if "minimum" in param_schema: + range_parts.append(f"min={param_schema['minimum']}") + if "maximum" in param_schema: + range_parts.append(f"max={param_schema['maximum']}") + parts.append(f"Range: {', '.join(range_parts)}") + if "pattern" in param_schema: + parts.append(f"Pattern: {param_schema['pattern']}") + return ". ".join(parts) if parts else "" diff --git a/litellm/llms/oci/embed/transformation.py b/litellm/llms/oci/embed/transformation.py index 1dcd8c5213ce..6cfa85b4bc4d 100644 --- a/litellm/llms/oci/embed/transformation.py +++ b/litellm/llms/oci/embed/transformation.py @@ -1,8 +1,14 @@ """ -OCI Generative AI Embedding Configuration +OCI Generative AI — Embedding transformation. -Supports embedding models available on Oracle Cloud Infrastructure Generative AI service. -Uses the same authentication mechanisms as OCI chat (manual signing or OCI SDK Signer). +Endpoint: POST /20231130/actions/embedText +Supported models: cohere.embed-english-v3.0, cohere.embed-multilingual-v3.0, +cohere.embed-v4.0, and all other Cohere embed variants available on OCI +(including dedicated endpoints). + +Authentication follows the same RSA-SHA256 / OCI SDK signer pattern as chat. +The base handler (base_llm_http_handler.embedding) calls sign_request after +building the body, so signing happens automatically. Supported models: - cohere.embed-english-v3.0 @@ -10,25 +16,45 @@ - cohere.embed-multilingual-v3.0 - cohere.embed-multilingual-light-v3.0 - cohere.embed-english-image-v3.0 -- cohere.embed-english-light-image-v3.0 -- cohere.embed-multilingual-light-image-v3.0 +- cohere.embed-multilingual-image-v3.0 - cohere.embed-v4.0 Reference: https://docs.oracle.com/en-us/iaas/api/#/en/generative-ai-inference/latest/EmbedTextResult/EmbedText """ -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union import httpx -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.llms.oci.chat.transformation import OCIChatConfig -from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.common_utils import ( + OCI_API_VERSION, + OCIError, + get_oci_base_url, + resolve_oci_credentials, + sign_oci_request, + validate_oci_environment, +) +from litellm.types.llms.oci import ( + OCIEmbedRequest, + OCIEmbedResponse, + OCIServingMode, +) from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +# OCI sends up to 96 texts per embedText request (Cohere limit). +OCI_EMBED_BATCH_LIMIT = 96 + # Input type mapping from OpenAI conventions to OCI/Cohere conventions _INPUT_TYPE_MAP = { "search_document": "SEARCH_DOCUMENT", @@ -38,65 +64,43 @@ } -class OCIEmbeddingConfig(BaseEmbeddingConfig): - """ - Configuration for OCI Generative AI Embedding API. - - The OCI embedding endpoint uses the Cohere embed models hosted on OCI. - Authentication is handled via OCI request signing (manual credentials or OCI SDK Signer). - - Usage: - ```python - import litellm - - response = litellm.embedding( - model="oci/cohere.embed-english-v3.0", - input=["Hello world", "Goodbye world"], - oci_compartment_id="ocid1.compartment.oc1..xxx", - oci_region="us-ashburn-1", - oci_user="ocid1.user.oc1..xxx", - oci_fingerprint="xx:xx:xx:xx", - oci_tenancy="ocid1.tenancy.oc1..xxx", - oci_key_file="~/.oci/key.pem", - ) - ``` +class OCIEmbedConfig(BaseEmbeddingConfig): """ + Transformation config for OCI Generative AI embeddings. - def __init__(self) -> None: - # We reuse OCIChatConfig for signing logic - self._chat_config = OCIChatConfig() + Supports both text and (on cohere.embed-v4.0) multimodal inputs. - def get_complete_url( - self, - api_base: Optional[str], - api_key: Optional[str], - model: str, - optional_params: dict, - litellm_params: dict, - stream: Optional[bool] = None, - ) -> str: - if api_base: - return api_base + Authentication — same two modes as chat: + - **OCI SDK signer**: pass ``oci_signer`` in optional_params. + - **Manual RSA-SHA256**: pass ``oci_user``, ``oci_fingerprint``, ``oci_tenancy``, + and ``oci_key`` or ``oci_key_file``, or set the corresponding ``OCI_*`` env vars. - oci_region = optional_params.get("oci_region", "us-ashburn-1") - return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/embedText" + Required call-time params (via optional_params or env vars): + - ``oci_compartment_id`` / ``OCI_COMPARTMENT_ID`` + - ``oci_region`` / ``OCI_REGION`` (default: ``us-ashburn-1``) - def get_supported_openai_params(self, model: str) -> list: - return [ - "dimensions", - ] + Optional call-time params: + - ``oci_serving_mode``: ``"ON_DEMAND"`` (default) or ``"DEDICATED"`` + - ``oci_endpoint_id``: endpoint OCID for dedicated serving mode + - ``input_type``: ``SEARCH_DOCUMENT``, ``SEARCH_QUERY``, ``CLASSIFICATION``, ``CLUSTERING`` + - ``truncate``: ``NONE``, ``START``, or ``END`` (default ``END``) + - ``dimensions``: output embedding dimensions (cohere.embed-v4.0+) + """ + + def get_supported_openai_params(self, model: str) -> List[str]: + return ["dimensions"] def map_openai_params( self, non_default_params: dict, optional_params: dict, model: str, - drop_params: bool, + drop_params: bool = False, ) -> dict: - # Note: OCI Cohere embed does not support custom dimensions natively, - # but we pass it through in case future models support it - if "dimensions" in non_default_params: - optional_params["dimensions"] = non_default_params["dimensions"] + for key, value in non_default_params.items(): + if key == "dimensions": + # OCI API uses outputDimensions (cohere.embed-v4.0+) + optional_params["outputDimensions"] = value return optional_params def validate_environment( @@ -109,49 +113,42 @@ def validate_environment( api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - """ - Validate OCI credentials for embedding requests. - Supports both OCI SDK Signer and manual credential signing. - """ - oci_signer = optional_params.get("oci_signer") - oci_region = optional_params.get("oci_region", "us-ashburn-1") - - api_base = ( - api_base - or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" - ) - - if oci_signer is None: - oci_user = optional_params.get("oci_user") - oci_fingerprint = optional_params.get("oci_fingerprint") - oci_tenancy = optional_params.get("oci_tenancy") - oci_key = optional_params.get("oci_key") - oci_key_file = optional_params.get("oci_key_file") - oci_compartment_id = optional_params.get("oci_compartment_id") - - if ( - not oci_user - or not oci_fingerprint - or not oci_tenancy - or not (oci_key or oci_key_file) - or not oci_compartment_id - ): - raise Exception( - "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " - "and at least one of oci_key or oci_key_file. " - "Alternatively, provide an oci_signer object from the OCI SDK." + if optional_params.get("oci_signer") is None: + creds = resolve_oci_credentials(optional_params) + missing = [ + k + for k in ( + "oci_user", + "oci_fingerprint", + "oci_tenancy", + "oci_compartment_id", ) + if not creds.get(k) + ] + if missing or not (creds.get("oci_key") or creds.get("oci_key_file")): + raise OCIError( + status_code=401, + message=( + "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, " + "oci_compartment_id and at least one of oci_key or oci_key_file. " + "These can be supplied via optional_params or via OCI_USER, OCI_FINGERPRINT, " + "OCI_TENANCY, OCI_COMPARTMENT_ID, OCI_KEY_FILE environment variables. " + "Alternatively, provide an oci_signer object from the OCI SDK." + ), + ) + return validate_oci_environment(headers, optional_params, api_key) - from litellm.llms.custom_httpx.http_handler import version - - headers.update( - { - "content-type": "application/json", - "user-agent": f"litellm/{version}", - } - ) - - return headers + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + base = get_oci_base_url(optional_params, api_base or litellm.api_base) + return f"{base}/{OCI_API_VERSION}/actions/embedText" def sign_request( self, @@ -163,9 +160,8 @@ def sign_request( model: Optional[str] = None, stream: Optional[bool] = None, fake_stream: Optional[bool] = None, - ): - """Delegate to OCIChatConfig's signing logic.""" - return self._chat_config.sign_request( + ) -> Tuple[dict, bytes]: + return sign_oci_request( headers=headers, optional_params=optional_params, request_data=request_data, @@ -182,91 +178,74 @@ def transform_embedding_request( input: AllEmbeddingInputValues, optional_params: dict, headers: dict, - api_base: Optional[str] = None, ) -> dict: - """ - Transform the embedding request to OCI format. - - OCI embedText API expects: - { - "compartmentId": "...", - "servingMode": {"servingType": "ON_DEMAND", "modelId": "..."}, - "inputs": ["text1", "text2"], - "truncate": "END", - "inputType": "SEARCH_DOCUMENT" - } - """ - oci_compartment_id = optional_params.get("oci_compartment_id") - if not oci_compartment_id: - raise Exception( - "kwarg `oci_compartment_id` is required for OCI embedding requests" + creds = resolve_oci_credentials(optional_params) + compartment_id = creds["oci_compartment_id"] + if not compartment_id: + raise OCIError( + status_code=400, + message=( + "oci_compartment_id is required for OCI embedding requests. " + "Pass it as optional_params or set the OCI_COMPARTMENT_ID env var." + ), ) - # Build serving mode - oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND") - if oci_serving_mode == "DEDICATED": - oci_endpoint_id = optional_params.get("oci_endpoint_id", model) - serving_mode = { - "servingType": "DEDICATED", - "endpointId": oci_endpoint_id, - } - else: - serving_mode = { - "servingType": "ON_DEMAND", - "modelId": model, - } - - # Normalize input to list of strings + # Normalise input to a flat list of strings if isinstance(input, str): - inputs = [input] + texts = [input] elif isinstance(input, list): - inputs = [] + texts = [] for item in input: - if isinstance(item, str): - inputs.append(item) - elif isinstance(item, list): - raise ValueError( - "OCI embedding does not support token-array inputs. " - "Please convert token lists to strings before calling embedding()." + if isinstance(item, list): + raise OCIError( + status_code=400, + message=( + "OCI embedText does not support token-array inputs. " + "Convert token lists to strings before calling embedding()." + ), ) - else: - inputs.append(str(item)) + texts.append(item if isinstance(item, str) else str(item)) else: - inputs = [str(input)] - - # Build request data — OCI embedText API expects inputs, truncate, - # and inputType at the top level alongside compartmentId and servingMode - request_data: Dict[str, Any] = { - "compartmentId": oci_compartment_id, - "servingMode": serving_mode, - "inputs": inputs, - "truncate": optional_params.get("truncate", "END"), - } - - # Map input_type if provided - input_type = optional_params.get("input_type") - if input_type: - mapped_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper()) - request_data["inputType"] = mapped_type + texts = [str(input)] - # Sign the request using the same URL the HTTP handler will POST to - signing_url = self.get_complete_url( - api_base=api_base, - api_key=None, - model=model, - optional_params=optional_params, - litellm_params={}, - ) + if len(texts) > OCI_EMBED_BATCH_LIMIT: + raise OCIError( + status_code=400, + message=( + f"OCI embedText accepts at most {OCI_EMBED_BATCH_LIMIT} inputs per request " + f"(got {len(texts)}). Batch your requests." + ), + ) - signed_headers, body = self.sign_request( - headers=headers, - optional_params=optional_params, - request_data=request_data, - api_base=signing_url, - ) - headers.update(signed_headers) + serving_mode_type = optional_params.get("oci_serving_mode", "ON_DEMAND").upper() + if serving_mode_type not in {"ON_DEMAND", "DEDICATED"}: + raise OCIError( + status_code=400, + message="oci_serving_mode must be 'ON_DEMAND' or 'DEDICATED'.", + ) + + if serving_mode_type == "DEDICATED": + endpoint_id = optional_params.get("oci_endpoint_id", model) + serving_mode = OCIServingMode( + servingType="DEDICATED", endpointId=endpoint_id + ) + else: + serving_mode = OCIServingMode(servingType="ON_DEMAND", modelId=model) - return request_data + # Map input_type from OpenAI convention to OCI/Cohere convention + input_type = optional_params.get("input_type") + if input_type: + input_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper()) + + request = OCIEmbedRequest( + compartmentId=compartment_id, + servingMode=serving_mode, + inputs=texts, + inputType=input_type, + truncate=optional_params.get("truncate", "END"), + outputDimensions=optional_params.get("outputDimensions"), + ) + return request.model_dump(exclude_none=True) def transform_embedding_response( self, @@ -274,63 +253,57 @@ def transform_embedding_response( raw_response: httpx.Response, model_response: EmbeddingResponse, logging_obj: LiteLLMLoggingObj, - api_key: Optional[str] = None, - request_data: dict = {}, - optional_params: dict = {}, - litellm_params: dict = {}, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, ) -> EmbeddingResponse: - """ - Transform OCI embedding response to standard EmbeddingResponse format. - - OCI response format: - { - "embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]], - "modelId": "cohere.embed-english-v3.0", - "modelVersion": "3.0", - "inputTextTokenCounts": [5, 4] - } - """ if raw_response.status_code != 200: raise OCIError( - message=raw_response.text, status_code=raw_response.status_code, + message=raw_response.text, ) try: - raw_response_json = raw_response.json() - except Exception: + json_response = raw_response.json() + except Exception as e: raise OCIError( - message=raw_response.text, status_code=raw_response.status_code, + message=f"Failed to parse OCI embed response as JSON: {e}", ) - embeddings = raw_response_json.get("embeddings", []) - model_id = raw_response_json.get("modelId", model) - - # Build response data in OpenAI format - embedding_data = [] - for idx, embedding in enumerate(embeddings): - embedding_data.append( - { - "object": "embedding", - "index": idx, - "embedding": embedding, - } + try: + parsed = OCIEmbedResponse(**json_response) + except Exception as e: + raise OCIError( + status_code=500, + message=f"OCI embed response does not match expected schema: {e}", ) - model_response.model = model_id - model_response.data = embedding_data - model_response.object = "list" - - # Calculate token usage - input_token_counts = raw_response_json.get("inputTextTokenCounts", []) - total_tokens = sum(input_token_counts) if input_token_counts else 0 + model_response.model = parsed.modelId + model_response.data = [ + { + "object": "embedding", + "index": i, + "embedding": embedding, + } + for i, embedding in enumerate(parsed.embeddings) + ] - usage = Usage( - prompt_tokens=total_tokens, - total_tokens=total_tokens, - ) - model_response.usage = usage + if parsed.inputTextTokenCounts is not None: + # Actual OCI API returns per-input token counts — sum for total usage + total = sum(parsed.inputTextTokenCounts) + model_response.usage = Usage(prompt_tokens=total, total_tokens=total) + elif parsed.usage is not None: + # Some deployments may return a usage object directly + model_response.usage = Usage( + prompt_tokens=parsed.usage.promptTokens, + total_tokens=parsed.usage.totalTokens, + ) + else: + # Neither field returned — default to zero so downstream consumers + # can always rely on usage being populated. + model_response.usage = Usage(prompt_tokens=0, total_tokens=0) return model_response @@ -340,8 +313,8 @@ def get_error_class( status_code: int, headers: Union[dict, httpx.Headers], ) -> BaseLLMException: - return OCIError( - message=error_message, - status_code=status_code, - headers=headers if isinstance(headers, httpx.Headers) else None, - ) + return OCIError(status_code=status_code, message=error_message) + + +# Alias for backwards compatibility with any code that imports OCIEmbeddingConfig +OCIEmbeddingConfig = OCIEmbedConfig diff --git a/litellm/main.py b/litellm/main.py index e17a5ad9a483..510e6e424a2e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5127,6 +5127,24 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, ) + elif custom_llm_provider == "oci": + if headers is None: + headers = {} + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params=litellm_params_dict, + headers=headers, + ) elif custom_llm_provider == "cohere" or custom_llm_provider == "cohere_chat": cohere_key = ( api_key @@ -5807,22 +5825,6 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) - elif custom_llm_provider == "oci": - response = base_llm_http_handler.embedding( - model=model, - input=input, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - api_key=api_key, - logging_obj=logging, - timeout=timeout, - model_response=EmbeddingResponse(), - optional_params=optional_params, - client=client, - aembedding=aembedding, - litellm_params=litellm_params_dict, - headers=headers, - ) elif custom_llm_provider in litellm._custom_providers: custom_handler: Optional[CustomLLM] = None for item in litellm.custom_provider_map: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6a4a5dd6a03e..08c06b17ddf8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26297,6 +26297,51 @@ "supports_function_calling": true, "supports_response_schema": false }, + "oci/openai.gpt-5": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/openai.gpt-5-mini": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/openai.gpt-5-nano": { + "input_cost_per_token": 5e-08, + "litellm_provider": "oci", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, "oci/google.gemini-2.5-pro": { "input_cost_per_token": 1.25e-06, "litellm_provider": "oci", diff --git a/litellm/router.py b/litellm/router.py index debccb0e83fe..2c61031da34a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10764,7 +10764,7 @@ async def async_get_healthy_deployments( if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) @@ -11028,7 +11028,7 @@ async def async_get_available_deployment_for_pass_through( if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) @@ -11202,7 +11202,7 @@ def get_available_deployment( if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) @@ -11359,7 +11359,7 @@ def get_available_deployment_for_pass_through( if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) diff --git a/litellm/types/llms/oci.py b/litellm/types/llms/oci.py index e041810158a4..df551d8a8c60 100644 --- a/litellm/types/llms/oci.py +++ b/litellm/types/llms/oci.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel +from pydantic import BaseModel, SerializeAsAny OCIRoles = Literal["SYSTEM", "USER", "ASSISTANT", "TOOL"] @@ -15,7 +15,6 @@ class OCIVendors(Enum): """ COHERE = "COHERE" - GEMINI = "GEMINI" GENERIC = "GENERIC" @@ -57,7 +56,7 @@ class OCIImageContentPart(OCIContentPart): class OCIToolCall(BaseModel): """Represents a tool call made by the model.""" - id: str + id: Optional[str] = None # absent in some provider responses (e.g. Google via OCI) type: Literal["FUNCTION"] = "FUNCTION" name: str arguments: str # Arguments should be a JSON-serialized string @@ -96,13 +95,22 @@ class OCIChatRequestPayload(BaseModel): isStream: bool = False numGenerations: Optional[int] = None maxTokens: Optional[int] = None + # GPT-5+ on OCI rejects maxTokens and requires maxCompletionTokens. + maxCompletionTokens: Optional[int] = None temperature: Optional[float] = None topP: Optional[float] = None stop: Optional[List[str]] = None seed: Optional[int] = None frequencyPenalty: Optional[float] = None presencePenalty: Optional[float] = None + # Reasoning-token budget knob (OCI: NONE/MINIMAL/LOW/MEDIUM/HIGH). + # Honoured by GPT-5 family, Gemini 2.5, Grok reasoning variants, + # Cohere Command-A-Reasoning. Ignored by non-reasoning models. + reasoningEffort: Optional[str] = None responseFormat: Optional[Dict[str, Any]] = None + toolChoice: Optional[Union[str, Dict[str, Any]]] = None + logitBias: Optional[Dict[str, Any]] = None + logProbs: Optional[int] = None class OCIServingMode(BaseModel): @@ -141,7 +149,9 @@ class OCIResponseUsage(BaseModel): """Token usage in the OCI response.""" promptTokens: int - completionTokens: int + # completionTokens may be absent for reasoning models when all the output + # budget is consumed by reasoning tokens before any visible content is produced. + completionTokens: Optional[int] = None totalTokens: int completionTokensDetails: Optional[OCICompletionTokenDetails] = None promptTokensDetails: Optional[OCIPromptTokensDetails] = None @@ -151,7 +161,9 @@ class OCIResponseChoice(BaseModel): """A completion choice in the OCI response.""" index: int - message: OCIMessage + # message is absent when a reasoning model exhausts max_tokens in the + # reasoning phase without producing any visible content. + message: Optional[OCIMessage] = None finishReason: Optional[str] = None logprobs: Optional[Dict[str, Any]] = None @@ -203,6 +215,7 @@ class CohereStreamChunk(BaseModel): text: Optional[str] = None chatHistory: Optional[List[CohereMessage]] = None finishReason: Optional[str] = None + toolCalls: Optional[List[CohereToolCall]] = None pad: Optional[str] = None index: Optional[int] = None @@ -234,10 +247,14 @@ class CohereSystemMessage(CohereMessage): class CohereToolMessage(CohereMessage): - """Tool message in Cohere chat.""" + """Tool message in Cohere chat. + + The OCI Cohere API represents tool results via a ``toolResults`` list on the + TOOL-role history entry — not via a ``toolCallId`` string. + """ role: Literal["TOOL"] = "TOOL" - toolCallId: str + toolResults: List[CohereToolResult] class CohereParameterDefinition(BaseModel): @@ -264,10 +281,14 @@ class CohereToolCall(BaseModel): class CohereToolResult(BaseModel): - """Result of a tool call.""" + """Result of a tool call. + + Matches the OCI SDK's CohereToolResult: each result carries the originating + tool call (name + parameters) and a list of output objects. + """ - callId: str - result: str + call: CohereToolCall + outputs: List[Dict[str, Any]] class CohereResponseFormat(BaseModel): @@ -297,7 +318,11 @@ class CohereChatRequest(BaseModel): apiFormat: Literal["COHERE"] = "COHERE" # Optional fields - chatHistory: Optional[List[CohereMessage]] = None + # ``SerializeAsAny`` preserves subclass-specific fields (e.g. ``toolResults`` + # on ``CohereToolMessage``) when this request is serialized via ``model_dump``. + # Without it, Pydantic v2 would serialize each element using the declared + # ``CohereMessage`` schema and silently drop subclass fields. + chatHistory: Optional[List[SerializeAsAny[CohereMessage]]] = None maxTokens: Optional[int] = None temperature: Optional[float] = None topP: Optional[float] = None @@ -307,7 +332,10 @@ class CohereChatRequest(BaseModel): stopSequences: Optional[List[str]] = None seed: Optional[int] = None tools: Optional[List[CohereTool]] = None - toolChoice: Optional[Union[str, Dict[str, Any]]] = None + # NOTE: OCI's Cohere chat endpoint does not accept ``toolChoice`` — see + # ``OCIChatConfig.openai_to_oci_cohere_param_map`` which marks + # ``tool_choice`` as unsupported. The field is intentionally absent here + # so it isn't silently dropped or surfaced as a supported feature. responseFormat: Optional[ Union[ CohereResponseTextFormat, @@ -364,9 +392,12 @@ class CohereChatResponse(BaseModel): # Required fields text: str apiFormat: Literal["COHERE"] = "COHERE" - finishReason: Literal[ - "COMPLETE", "ERROR_TOXIC", "ERROR_LIMIT", "ERROR", "USER_CANCEL", "MAX_TOKENS" - ] + # Accept any string (with ``None`` for absent) so unknown finish reasons + # — e.g. a value OCI adds in a future API revision — degrade gracefully + # via ``handle_cohere_response``'s ``elif oci_finish_reason is not None`` + # fallback instead of crashing Pydantic validation. Mirrors + # ``CohereStreamChunk.finishReason`` which has always been ``Optional[str]``. + finishReason: Optional[str] = None # Optional fields chatHistory: Optional[List[CohereMessage]] = None @@ -394,3 +425,41 @@ class CohereChatResult(BaseModel): modelId: str modelVersion: str chatResponse: CohereChatResponse + + +# --------------------------------------------------------------------------- +# OCI Embed types +# --------------------------------------------------------------------------- + + +class OCIEmbedRequest(BaseModel): + """Request body for POST /20231130/actions/embedText.""" + + compartmentId: str + servingMode: OCIServingMode + inputs: List[str] + inputType: Optional[str] = ( + None # SEARCH_DOCUMENT | SEARCH_QUERY | CLASSIFICATION | CLUSTERING | IMAGE + ) + truncate: Optional[str] = "END" # NONE | START | END + outputDimensions: Optional[int] = ( + None # cohere.embed-v4.0+; valid: 256, 512, 1024, 1536 + ) + + +class OCIEmbedUsage(BaseModel): + promptTokens: int + totalTokens: int + + +class OCIEmbedResponse(BaseModel): + """Response body from POST /20231130/actions/embedText.""" + + id: Optional[str] = None # present in the official SDK response + embeddings: List[List[float]] + modelId: str + modelVersion: str + # OCI returns per-input token counts in inputTextTokenCounts (summed for total usage) + inputTextTokenCounts: Optional[List[int]] = None + # Some deployments may return a usage object instead + usage: Optional[OCIEmbedUsage] = None diff --git a/litellm/utils.py b/litellm/utils.py index c28a88e0f1c7..2a94941f6563 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8452,6 +8452,10 @@ def get_provider_embedding_config( return litellm.InfinityEmbeddingConfig() elif litellm.LlmProviders.SAMBANOVA == provider: return litellm.SambaNovaEmbeddingConfig() + elif litellm.LlmProviders.OCI == provider: + from litellm.llms.oci.embed.transformation import OCIEmbedConfig + + return OCIEmbedConfig() elif ( litellm.LlmProviders.COHERE == provider or litellm.LlmProviders.COHERE_CHAT == provider @@ -8509,10 +8513,6 @@ def get_provider_embedding_config( return SagemakerEmbeddingConfig.get_model_config(model) elif litellm.LlmProviders.PERPLEXITY == provider: return litellm.PerplexityEmbeddingConfig() - elif litellm.LlmProviders.OCI == provider: - from litellm.llms.oci.embed.transformation import OCIEmbeddingConfig - - return OCIEmbeddingConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2140493ec4a0..514862516e82 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26018,6 +26018,32 @@ "supports_vision": true, "supports_web_search": true }, + "oci/meta.llama-3.1-8b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, "oci/meta.llama-3.1-405b-instruct": { "input_cost_per_token": 1.068e-05, "litellm_provider": "oci", @@ -26028,7 +26054,8 @@ "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-3.2-90b-vision-instruct": { "input_cost_per_token": 2e-06, @@ -26041,6 +26068,7 @@ "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, "supports_response_schema": false, + "supports_native_streaming": true, "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { @@ -26053,31 +26081,35 @@ "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-4-maverick-17b-128e-instruct-fp8": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 512000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true }, "oci/meta.llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 192000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 10485760, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3": { "input_cost_per_token": 3e-06, @@ -26089,7 +26121,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-fast": { "input_cost_per_token": 5e-06, @@ -26101,7 +26134,8 @@ "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini": { "input_cost_per_token": 3e-07, @@ -26113,7 +26147,8 @@ "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini-fast": { "input_cost_per_token": 6e-07, @@ -26125,7 +26160,8 @@ "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-4": { "input_cost_per_token": 3e-06, @@ -26137,7 +26173,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-latest": { "input_cost_per_token": 1.56e-06, @@ -26149,7 +26186,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-a-03-2025": { "input_cost_per_token": 1.56e-06, @@ -26161,7 +26199,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-plus-latest": { "input_cost_per_token": 1.56e-06, @@ -26173,7 +26212,86 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/cohere.command-a-vision": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true + }, + "oci/cohere.command-a-reasoning": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": false, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/cohere.embed-multilingual-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "mode": "embedding", + "output_vector_size": 1024, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { "input_cost_per_token": 1.56e-06, @@ -26249,18 +26367,6 @@ "supports_response_schema": false, "supports_vision": true }, - "oci/meta.llama-3.1-70b-instruct": { - "input_cost_per_token": 7.2e-07, - "litellm_provider": "oci", - "max_input_tokens": 128000, - "max_output_tokens": 4000, - "max_tokens": 4000, - "mode": "chat", - "output_cost_per_token": 7.2e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": false - }, "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -26333,42 +26439,48 @@ "supports_function_calling": true, "supports_response_schema": false }, - "oci/google.gemini-2.5-pro": { + "oci/openai.gpt-5": { "input_cost_per_token": 1.25e-06, "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true }, - "oci/google.gemini-2.5-flash": { - "input_cost_per_token": 1.5e-07, + "oci/openai.gpt-5-mini": { + "input_cost_per_token": 2.5e-07, "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true }, - "oci/google.gemini-2.5-flash-lite": { - "input_cost_per_token": 7.5e-08, + "oci/openai.gpt-5-nano": { + "input_cost_per_token": 5e-08, "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true }, diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 07af2735dfe8..254d700ee5a7 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -47,6 +47,9 @@ "_read_image_bytes", # max depth set. "_get_masked_values", # max depth set (default 20) to prevent infinite recursion while masking nested sensitive config dicts. "_redact_sensitive_litellm_params", # max depth set (default 10). + "_resolve", # OCI: $ref resolver bounded by `resolving_stack` cycle guard. + "resolve_oci_schema_anyof", # OCI: bounded by JSON-schema tree depth (no cycles possible in well-formed input). + "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. ] diff --git a/tests/integration/oci_proxy_test_config.yaml b/tests/integration/oci_proxy_test_config.yaml new file mode 100644 index 000000000000..bccb58d27f3e --- /dev/null +++ b/tests/integration/oci_proxy_test_config.yaml @@ -0,0 +1,24 @@ +model_list: + - model_name: oci-cohere-command + litellm_params: + model: oci/cohere.command-latest + - model_name: oci-llama + litellm_params: + model: oci/meta.llama-3.3-70b-instruct + - model_name: oci-gemini + litellm_params: + model: oci/google.gemini-2.5-flash + - model_name: oci-grok + litellm_params: + model: oci/xai.grok-3-mini + - model_name: oci-embed + litellm_params: + model: oci/cohere.embed-v4.0 + model_info: + mode: embedding + +general_settings: + master_key: sk-1234 + +litellm_settings: + drop_params: True diff --git a/tests/integration/test_oci_integration.py b/tests/integration/test_oci_integration.py new file mode 100644 index 000000000000..94b8930bce84 --- /dev/null +++ b/tests/integration/test_oci_integration.py @@ -0,0 +1,669 @@ +""" +OCI Generative AI — end-to-end integration tests. + +These tests make REAL calls to OCI. They are skipped automatically when the +standard ~/.oci/config is absent or when OCI_TEST_COMPARTMENT_ID is not set. + +Prerequisites +------------- +- ~/.oci/config with a valid [DEFAULT] profile +- Private key referenced by key_file in that profile +- Sufficient IAM policies to call the Generative AI inference service + +Environment variables (all optional — fall back to ~/.oci/config values): + OCI_TEST_REGION OCI region (default: us-chicago-1) + OCI_TEST_COMPARTMENT_ID compartment OCID (default: tenancy root from config) + +Run only these tests: + pytest tests/integration/test_oci_integration.py -v +""" + +import math +import os +import sys +from typing import NamedTuple, Optional + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + +OCI_CONFIG_FILE = os.path.expanduser("~/.oci/config") +_OCI_AVAILABLE = os.path.isfile(OCI_CONFIG_FILE) + +pytestmark = pytest.mark.skipif( + not _OCI_AVAILABLE, + reason="~/.oci/config not found — skipping OCI integration tests", +) + + +def _load_oci_config(): + """Load OCI config from the profile named by ``OCI_CONFIG_PROFILE`` env var, + falling back to ``[DEFAULT]``. Lets CI/local runs target a specific profile + without needing a ``[DEFAULT]`` section in ``~/.oci/config``.""" + oci = pytest.importorskip("oci") + profile = os.environ.get("OCI_CONFIG_PROFILE", "DEFAULT") + return oci.config.from_file(profile_name=profile) + + +@pytest.fixture(scope="module") +def oci_signer(): + """Return an oci.Signer (or SecurityTokenSigner for session-token profiles) + built from ~/.oci/config — profile chosen via OCI_CONFIG_PROFILE.""" + oci = pytest.importorskip("oci") + config = _load_oci_config() + + # Session-token profiles carry a `security_token_file` instead of a user + # OCID; build the corresponding signer in that case. + if "security_token_file" in config: + with open(os.path.expanduser(config["security_token_file"])) as f: + token = f.read().strip() + private_key = oci.signer.load_private_key_from_file( + config["key_file"], config.get("pass_phrase") + ) + return oci.auth.signers.SecurityTokenSigner(token, private_key) + + return oci.Signer( + tenancy=config["tenancy"], + user=config["user"], + fingerprint=config["fingerprint"], + private_key_file_location=config["key_file"], + ) + + +@pytest.fixture(scope="module") +def oci_params(oci_signer) -> dict: + """Common OCI call-time parameters shared by all tests.""" + config = _load_oci_config() + compartment_id = os.environ.get("OCI_TEST_COMPARTMENT_ID", config["tenancy"]) + region = os.environ.get("OCI_TEST_REGION", "us-chicago-1") + return { + "oci_signer": oci_signer, + "oci_compartment_id": compartment_id, + "oci_region": region, + } + + +# --------------------------------------------------------------------------- +# Model registry +# +# Each entry drives the runtime pivot inside OCI's own transformation layer — +# the tests themselves are format-agnostic. Per-model quirks are captured in +# the config fields below rather than in separate test classes. +# --------------------------------------------------------------------------- + + +class _M(NamedTuple): + """Per-model test configuration.""" + + model: str + max_tokens: int + # Reasoning models (Gemini 2.5, Grok mini) may return None content when the + # reasoning budget is exhausted before the answer token budget starts. + reasoning: bool = False + # tool_choice value to send; None means omit the parameter entirely. + tool_choice: Optional[str] = "auto" + # Whether to include the model in tool-use parametrize list. + supports_tool_use: bool = True + + +# All chat models under test. +CHAT_MODELS = [ + pytest.param(_M("meta.llama-3.3-70b-instruct", 64), id="meta"), + pytest.param(_M("google.gemini-2.5-flash", 200, reasoning=True), id="google"), + pytest.param(_M("xai.grok-3-mini", 100, reasoning=True), id="xai"), + pytest.param(_M("cohere.command-latest", 64, tool_choice=None), id="cohere"), +] + +# Subset of models that reliably support tool use in OCI. +# xAI Grok mini is omitted — OCI does not expose tool-use for it yet. +TOOL_USE_MODELS = [ + pytest.param(_M("meta.llama-3.3-70b-instruct", 100), id="meta"), + pytest.param(_M("cohere.command-latest", 200, tool_choice=None), id="cohere"), + pytest.param(_M("google.gemini-2.5-flash", 200, reasoning=True), id="google"), +] + +# Simple weather tool used by all tool-use tests. +_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string", "description": "The city name."}}, + "required": ["city"], + }, + }, +} + + +# --------------------------------------------------------------------------- +# Sync chat tests — model list drives the pivot, not separate test classes +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_basic_completion(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Reply with only the word: pong"}], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + assert resp.usage.prompt_tokens > 0 + if not m.reasoning: + assert resp.choices[0].message.content is not None + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_usage_populated(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "What is 2+2?"}], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.usage.prompt_tokens > 0 + assert resp.usage.total_tokens >= resp.usage.prompt_tokens + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_system_message(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Say hello."}, + ], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_streaming(m: _M, oci_params): + import litellm + + chunks = list( + litellm.completion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Count to 3."}], + max_tokens=m.max_tokens, + stream=True, + **oci_params, + ) + ) + assert len(chunks) > 0 + # Reasoning models may stream only reasoning tokens and return empty content. + if not m.reasoning: + content = "".join(c.choices[0].delta.content or "" for c in chunks if c.choices) + assert len(content) > 0 + + +@pytest.mark.parametrize( + "model", + ["cohere.command-latest", "cohere.command-r-plus-08-2024"], +) +def test_cohere_streaming_no_doubling(model, oci_params): + """Regression: OCI Cohere's terminal SSE event re-sends the full assembled + response in `text` alongside a populated `chatHistory`. Emitting that text + as another delta would concatenate the whole response onto the + already-streamed output (e.g. "How can I help?How can I help?"). + + Reported by @gotsysdba on PR #25177. Fix: drop terminal text when + `chatHistory` is present in `handle_cohere_stream_chunk`. + """ + import litellm + + streamed = "".join( + (c.choices[0].delta.content or "") + for c in litellm.completion( + model=f"oci/{model}", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=64, + stream=True, + **oci_params, + ) + if c.choices + ).strip() + + assert streamed, "expected non-empty streamed content" + + # Compare against a non-streamed call. With the doubling bug the streamed + # assembly is ~2x the real response; without it the two are the same order + # of magnitude (the model is non-deterministic, so allow generous slack). + non_streamed = ( + litellm.completion( + model=f"oci/{model}", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=64, + **oci_params, + ) + .choices[0] + .message.content + or "" + ).strip() + + assert len(streamed) < 2 * len(non_streamed) + 10, ( + f"streamed output appears doubled — " + f"streamed={len(streamed)} chars vs non_streamed={len(non_streamed)} chars\n" + f"streamed: {streamed!r}\n" + f"non_streamed: {non_streamed!r}" + ) + + # Stronger signal: the very start of the response should not appear twice. + head = streamed[:12] + assert streamed.count(head) == 1, ( + f"streamed output contains its own prefix {head!r} more than once — " + f"likely the terminal chunk re-emitted the full response.\n" + f"streamed: {streamed!r}" + ) + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_multi_turn(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[ + {"role": "user", "content": "My name is Alice."}, + {"role": "assistant", "content": "Nice to meet you, Alice!"}, + {"role": "user", "content": "What is my name?"}, + ], + max_tokens=m.max_tokens, + **oci_params, + ) + # Reasoning models may have None content; skip text assertion for them. + content = resp.choices[0].message.content or "" + if not m.reasoning: + assert "Alice" in content + + +# --------------------------------------------------------------------------- +# Async chat tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("m", CHAT_MODELS) +async def test_async_completion(m: _M, oci_params): + import litellm + + resp = await litellm.acompletion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Reply with only the word: pong"}], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + assert resp.usage.total_tokens > 0 + if not m.reasoning: + assert resp.choices[0].message.content is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("m", CHAT_MODELS) +async def test_async_streaming(m: _M, oci_params): + import litellm + + chunks = [] + async for chunk in await litellm.acompletion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Count to 3."}], + max_tokens=m.max_tokens, + stream=True, + **oci_params, + ): + chunks.append(chunk) + + assert len(chunks) > 0 + if not m.reasoning: + content = "".join(c.choices[0].delta.content or "" for c in chunks if c.choices) + assert len(content) > 0 + + +# --------------------------------------------------------------------------- +# Tool-use tests +# --------------------------------------------------------------------------- + + +def _assert_tool_call(resp, expected_tool: str = "get_weather"): + """Assert the response contains the expected tool call (or a plain stop).""" + choice = resp.choices[0] + assert choice.finish_reason in ("tool_calls", "stop") + if choice.finish_reason == "tool_calls": + assert choice.message.tool_calls is not None + assert len(choice.message.tool_calls) > 0 + assert choice.message.tool_calls[0].function.name == expected_tool + + +@pytest.mark.parametrize("m", TOOL_USE_MODELS) +def test_tool_use(m: _M, oci_params): + import litellm + + call_kwargs = dict( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + max_tokens=m.max_tokens, + **oci_params, + ) + if m.tool_choice is not None: + call_kwargs["tool_choice"] = m.tool_choice + + resp = litellm.completion(**call_kwargs) + _assert_tool_call(resp) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("m", TOOL_USE_MODELS) +async def test_async_tool_use(m: _M, oci_params): + import litellm + + call_kwargs = dict( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "What is the weather in Berlin?"}], + tools=[_WEATHER_TOOL], + max_tokens=m.max_tokens, + **oci_params, + ) + if m.tool_choice is not None: + call_kwargs["tool_choice"] = m.tool_choice + + resp = await litellm.acompletion(**call_kwargs) + _assert_tool_call(resp) + + +# --------------------------------------------------------------------------- +# Reasoning-effort tests (reasoning models only) +# --------------------------------------------------------------------------- + +# Reasoning model that accepts the `reasoningEffort` parameter on OCI. +# Not every reasoning model does — xai.grok-4-fast-reasoning, for example, +# rejects it with a 400. +_REASONING_MODEL = "xai.grok-3-mini" + + +@pytest.mark.parametrize("effort", ["low", "medium", "high"]) +def test_reasoning_effort_lowercase_accepted(effort, oci_params): + """OpenAI clients send lowercase reasoning_effort; OCI requires uppercase. + The transform layer should uppercase it transparently.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort=effort, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + assert resp.usage.prompt_tokens > 0 + + +def test_reasoning_effort_disable_mapped_to_none(oci_params): + """OpenAI's 'disable' maps to OCI's 'NONE'. Without this mapping the + request 400s.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort="disable", + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + + +def test_reasoning_tokens_in_usage(oci_params): + """OCI returns completionTokensDetails.reasoningTokens on reasoning models; + LiteLLM should surface it on Usage.completion_tokens_details.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort="low", + **oci_params, + ) + assert resp.usage.completion_tokens_details is not None + assert resp.usage.completion_tokens_details.reasoning_tokens is not None + assert resp.usage.completion_tokens_details.reasoning_tokens > 0 + + +# --------------------------------------------------------------------------- +# Embedding tests +# --------------------------------------------------------------------------- + + +class TestOCIEmbeddings: + + def test_english_v3_basic(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 1 + assert len(resp.data[0]["embedding"]) == 1024 + assert resp.usage.prompt_tokens > 0 + + def test_english_v3_batch(self, oci_params): + import litellm + + texts = [ + "The quick brown fox", + "jumps over the lazy dog", + "Paris is the capital of France", + ] + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=texts, + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 3 + for i, item in enumerate(resp.data): + assert item["index"] == i + assert len(item["embedding"]) == 1024 + + def test_multilingual_v3(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-multilingual-v3.0", + input=["Bonjour le monde", "Hola mundo"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 2 + assert len(resp.data[0]["embedding"]) == 1024 + + def test_search_query_input_type(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["What is the capital of France?"], + input_type="SEARCH_QUERY", + **oci_params, + ) + assert len(resp.data[0]["embedding"]) == 1024 + + def test_semantic_similarity(self, oci_params): + """Semantically similar texts should have higher cosine similarity.""" + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=[ + "The cat sat on the mat", + "A feline rested on the rug", + "The stock market crashed today", + ], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + + def cosine(a, b): + dot = sum(x * y for x, y in zip(a, b)) + mag_a = math.sqrt(sum(x**2 for x in a)) + mag_b = math.sqrt(sum(x**2 for x in b)) + return dot / (mag_a * mag_b) + + cat1 = resp.data[0]["embedding"] + cat2 = resp.data[1]["embedding"] + stock = resp.data[2]["embedding"] + sim_cats = cosine(cat1, cat2) + sim_diff = cosine(cat1, stock) + assert ( + sim_cats > sim_diff + ), f"Expected similar sentences to score higher ({sim_cats:.3f} vs {sim_diff:.3f})" + + def test_embed_v4(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-v4.0", + input=["Hello world"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 1 + assert len(resp.data[0]["embedding"]) == 1536 + + def test_usage_tokens(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["short text", "another short text"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert resp.usage.prompt_tokens > 0 + assert resp.usage.total_tokens == resp.usage.prompt_tokens + + +# --------------------------------------------------------------------------- +# Async embedding tests +# --------------------------------------------------------------------------- + + +class TestOCIAsyncEmbeddings: + + @pytest.mark.asyncio + async def test_async_embedding_basic(self, oci_params): + import litellm + + resp = await litellm.aembedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 1 + assert len(resp.data[0]["embedding"]) == 1024 + assert resp.usage.prompt_tokens > 0 + + @pytest.mark.asyncio + async def test_async_embedding_batch(self, oci_params): + import litellm + + texts = ["The quick brown fox", "jumps over the lazy dog"] + resp = await litellm.aembedding( + model="oci/cohere.embed-english-v3.0", + input=texts, + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 2 + assert all(len(item["embedding"]) == 1024 for item in resp.data) + + @pytest.mark.asyncio + async def test_async_embedding_multilingual(self, oci_params): + import litellm + + resp = await litellm.aembedding( + model="oci/cohere.embed-multilingual-v3.0", + input=["Bonjour le monde"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data[0]["embedding"]) == 1024 + + +# --------------------------------------------------------------------------- +# Env-var credential path +# --------------------------------------------------------------------------- + + +class TestOCIEnvVarCredentials: + """Verify that OCI_* env vars are picked up without explicit params.""" + + def test_completion_via_env_vars(self, monkeypatch): + """Completion works when credentials are set through environment variables.""" + pytest.importorskip("oci") + config = _load_oci_config() + key_path = os.path.expanduser(config["key_file"]) + + with open(key_path) as f: + key_pem = f.read() + + monkeypatch.setenv("OCI_REGION", "us-chicago-1") + monkeypatch.setenv("OCI_USER", config["user"]) + monkeypatch.setenv("OCI_FINGERPRINT", config["fingerprint"]) + monkeypatch.setenv("OCI_TENANCY", config["tenancy"]) + monkeypatch.setenv("OCI_KEY", key_pem) + monkeypatch.setenv("OCI_COMPARTMENT_ID", config["tenancy"]) + + import litellm + + resp = litellm.completion( + model="oci/meta.llama-3.3-70b-instruct", + messages=[{"role": "user", "content": "Reply with only the word: pong"}], + max_tokens=10, + ) + assert resp.choices[0].message.content is not None + + def test_embedding_via_env_vars(self, monkeypatch): + pytest.importorskip("oci") + config = _load_oci_config() + key_path = os.path.expanduser(config["key_file"]) + + with open(key_path) as f: + key_pem = f.read() + + monkeypatch.setenv("OCI_REGION", "us-chicago-1") + monkeypatch.setenv("OCI_USER", config["user"]) + monkeypatch.setenv("OCI_FINGERPRINT", config["fingerprint"]) + monkeypatch.setenv("OCI_TENANCY", config["tenancy"]) + monkeypatch.setenv("OCI_KEY", key_pem) + monkeypatch.setenv("OCI_COMPARTMENT_ID", config["tenancy"]) + + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["hello"], + input_type="SEARCH_DOCUMENT", + ) + assert len(resp.data[0]["embedding"]) == 1024 diff --git a/tests/integration/test_oci_proxy_integration.py b/tests/integration/test_oci_proxy_integration.py new file mode 100644 index 000000000000..8bfcdd904862 --- /dev/null +++ b/tests/integration/test_oci_proxy_integration.py @@ -0,0 +1,274 @@ +""" +OCI GenAI — end-to-end **proxy** integration tests. + +Spins up the LiteLLM proxy (`litellm --config oci_proxy_test_config.yaml`) as a +subprocess, then sends OpenAI-shaped HTTP requests at it for the OCI models +declared in the test config: + + - oci-cohere-command (oci/cohere.command-latest) + - oci-llama (oci/meta.llama-3.3-70b-instruct) + - oci-gemini (oci/google.gemini-2.5-flash) + - oci-grok (oci/xai.grok-3-mini) + - oci-embed (oci/cohere.embed-v4.0) + +Skipped unless: + - ~/.oci/config exists + - The `oci` SDK is installed (handled by ``pytest.importorskip``) + +Environment variables honoured (passed through to the proxy subprocess): + OCI_CONFIG_PROFILE profile inside ~/.oci/config (default: DEFAULT) + OCI_REGION overrides region from the profile (default: us-chicago-1) + +Run with:: + + OCI_CONFIG_PROFILE=LUIGI_FRA_API OCI_REGION=us-chicago-1 \ + uv run pytest tests/integration/test_oci_proxy_integration.py -v -s + +The tests open a real socket on a free TCP port — no port collision with a +locally-running proxy. +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import Iterator + +import httpx +import pytest + + +# --------------------------------------------------------------------------- +# Skip gate +# --------------------------------------------------------------------------- +OCI_CONFIG_FILE = os.path.expanduser("~/.oci/config") +pytestmark = pytest.mark.skipif( + not os.path.isfile(OCI_CONFIG_FILE), + reason="~/.oci/config not found — skipping OCI proxy integration tests", +) + + +CONFIG_PATH = Path(__file__).parent / "oci_proxy_test_config.yaml" +MASTER_KEY = "sk-1234" +STARTUP_TIMEOUT_S = 90.0 +REQUEST_TIMEOUT_S = 120.0 + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_for_health(base_url: str, proc: subprocess.Popen, deadline: float) -> None: + """Poll /health/liveliness until the proxy answers or the deadline expires.""" + while time.monotonic() < deadline: + if proc.poll() is not None: + output = proc.stdout.read() if proc.stdout else "" + raise RuntimeError( + f"litellm proxy exited early with code {proc.returncode}\n--- proxy output ---\n{output}" + ) + try: + r = httpx.get(f"{base_url}/health/liveliness", timeout=2.0) + if r.status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(0.5) + raise RuntimeError(f"litellm proxy did not become ready within {STARTUP_TIMEOUT_S}s") + + +def _oci_env_from_profile() -> dict[str, str]: + """Translate the active OCI profile into the OCI_* env vars the litellm + OCI provider expects. Only API-key profiles are supported; session-token + profiles would need an in-process signer and so are skipped here. + """ + oci = pytest.importorskip("oci") + profile = os.environ.get("OCI_CONFIG_PROFILE", "DEFAULT") + cfg = oci.config.from_file(profile_name=profile) + if "security_token_file" in cfg: + pytest.skip( + f"OCI profile {profile!r} uses session-token auth; " + "litellm's OCI provider needs an API-key profile for env-driven config" + ) + region = os.environ.get("OCI_REGION") or cfg.get("region") or "us-chicago-1" + return { + "OCI_USER": cfg["user"], + "OCI_FINGERPRINT": cfg["fingerprint"], + "OCI_TENANCY": cfg["tenancy"], + "OCI_COMPARTMENT_ID": os.environ.get("OCI_COMPARTMENT_ID", cfg["tenancy"]), + "OCI_KEY_FILE": os.path.expanduser(cfg["key_file"]), + "OCI_REGION": region, + } + + +@pytest.fixture(scope="module") +def proxy_url() -> Iterator[str]: + oci_env = _oci_env_from_profile() + + port = _free_port() + base_url = f"http://127.0.0.1:{port}" + + env = os.environ.copy() + env.update(oci_env) + # Avoid pulling in DB-backed features for this lightweight smoke run. + env.pop("DATABASE_URL", None) + env["STORE_MODEL_IN_DB"] = "False" + + # Prefer the `litellm` console script that lives next to the active + # Python so we inherit the test virtualenv. Fall back to PATH. + cli = Path(sys.executable).parent / "litellm" + if not cli.exists(): + cli = "litellm" + cmd = [ + str(cli), + "--config", + str(CONFIG_PATH), + "--port", + str(port), + "--host", + "127.0.0.1", + "--num_workers", + "1", + ] + + proc = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + _wait_for_health(base_url, proc, time.monotonic() + STARTUP_TIMEOUT_S) + yield base_url + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _auth_headers() -> dict: + return { + "Authorization": f"Bearer {MASTER_KEY}", + "Content-Type": "application/json", + } + + +def _chat_payload(model: str, *, stream: bool = False) -> dict: + return { + "model": model, + "messages": [ + {"role": "user", "content": "Reply with only the single word: pong"} + ], + "max_tokens": 64, + "stream": stream, + } + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +CHAT_MODELS = ["oci-cohere-command", "oci-llama", "oci-gemini", "oci-grok"] + + +@pytest.mark.parametrize("model", CHAT_MODELS) +def test_chat_completion_via_proxy(proxy_url: str, model: str) -> None: + """Non-streaming chat completion returns a well-formed OpenAI response.""" + r = httpx.post( + f"{proxy_url}/v1/chat/completions", + headers=_auth_headers(), + json=_chat_payload(model), + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, f"{model} -> {r.status_code}: {r.text}" + body = r.json() + assert body["object"] == "chat.completion" + assert body["model"] == model + choices = body["choices"] + assert len(choices) >= 1 + msg = choices[0]["message"] + assert msg["role"] == "assistant" + # Reasoning models may return empty content if their budget covers only + # the thinking turn — accept either text or a non-empty reasoning field. + has_content = bool(msg.get("content")) + has_reasoning = bool(msg.get("reasoning_content")) or bool( + msg.get("reasoning") + ) + assert has_content or has_reasoning, f"empty assistant message for {model}: {msg}" + usage = body.get("usage") or {} + assert usage.get("total_tokens", 0) > 0 + + +@pytest.mark.parametrize("model", CHAT_MODELS) +def test_chat_completion_streaming_via_proxy(proxy_url: str, model: str) -> None: + """Streaming chat completion yields at least one data: chunk and a [DONE].""" + saw_chunk = False + saw_done = False + with httpx.stream( + "POST", + f"{proxy_url}/v1/chat/completions", + headers=_auth_headers(), + json=_chat_payload(model, stream=True), + timeout=REQUEST_TIMEOUT_S, + ) as r: + assert r.status_code == 200, f"{model} stream -> {r.status_code}: {r.read()!r}" + for line in r.iter_lines(): + if not line: + continue + if not line.startswith("data:"): + continue + payload = line[len("data:"):].strip() + if payload == "[DONE]": + saw_done = True + break + saw_chunk = True + assert saw_chunk, f"no streamed chunks for {model}" + assert saw_done, f"no [DONE] sentinel for {model}" + + +def test_embedding_via_proxy(proxy_url: str) -> None: + """OCI Cohere embedding endpoint returns a non-empty vector via the proxy.""" + r = httpx.post( + f"{proxy_url}/v1/embeddings", + headers=_auth_headers(), + json={"model": "oci-embed", "input": ["hello from the litellm proxy"]}, + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, f"embed -> {r.status_code}: {r.text}" + body = r.json() + assert body["object"] == "list" + assert body["model"] == "oci-embed" + data = body["data"] + assert len(data) == 1 + embedding = data[0]["embedding"] + assert isinstance(embedding, list) + assert len(embedding) >= 64 + assert all(isinstance(x, (int, float)) for x in embedding) + + +def test_model_list_advertises_oci_models(proxy_url: str) -> None: + """The /v1/models registry advertises every OCI alias from the config.""" + r = httpx.get( + f"{proxy_url}/v1/models", + headers=_auth_headers(), + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, r.text + advertised = {row["id"] for row in r.json()["data"]} + for expected in CHAT_MODELS + ["oci-embed"]: + assert expected in advertised, f"{expected} missing from /v1/models: {advertised}" diff --git a/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py index f96228a4ccc8..e9b3f82d1a74 100644 --- a/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -14,7 +14,7 @@ ) # Adds the parent directory to the system path from litellm.llms.oci.chat.transformation import OCIChatConfig -from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.common_utils import OCIError, sign_with_manual_credentials @pytest.fixture @@ -41,7 +41,7 @@ def test_oci_key_with_escaped_newlines(self, config): # We can't fully test signing without a real key, but we can verify # the error message indicates the key was processed (not a type error) with pytest.raises(Exception) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, @@ -67,7 +67,7 @@ def test_oci_key_with_crlf_newlines(self, config): } with pytest.raises(Exception) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, @@ -88,7 +88,7 @@ def test_oci_key_rejects_non_string_type(self, config): } with pytest.raises(OCIError) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, @@ -110,7 +110,7 @@ def test_oci_key_rejects_list_type(self, config): } with pytest.raises(OCIError) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 2a5cc6b8e3d5..e0911e1ef31d 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -443,7 +443,7 @@ def test_transform_response_simple_text(self): {"type": "TEXT", "text": "I am doing well, thank you!"} ], }, - "finishReason": "STOP", + "finishReason": "COMPLETE", } ], "timeCreated": created_time, @@ -490,9 +490,10 @@ def test_transform_response_simple_text(self): assert result.usage.prompt_tokens == 10 # type: ignore assert result.usage.completion_tokens == 20 # type: ignore assert result.usage.total_tokens == 30 # type: ignore - # These are not handled in the transformer, TBH no idea why they are here - # but, for now, they seem to be always None - assert result.usage.completion_tokens_details is None + # reasoningTokens from OCI's completionTokensDetails is surfaced on + # Usage.completion_tokens_details.reasoning_tokens. + assert result.usage.completion_tokens_details is not None + assert result.usage.completion_tokens_details.reasoning_tokens == 20 assert result.usage.prompt_tokens_details is None def test_transform_response_with_tool_calls(self): @@ -765,3 +766,557 @@ def test_oci_request_wrapper_path_url_no_query(self): ) assert wrapper.path_url == "/api/v1/chat" + + +class TestOCISplitChunks: + """ + Unit tests for the SSE split_chunks helpers used in sync and async streaming. + + These validate the fix for: + - Sync: JSONDecodeError when iter_text() returns chunks spanning multiple events + - Async: whitespace-only chunks being yielded before stripping (Greptile P2) + """ + + def _run_sync_split(self, raw_chunks): + """Invoke the sync split_chunks logic directly (extracted for testability).""" + results = [] + for item in raw_chunks: + for chunk in item.split("\n\n"): + stripped = chunk.strip() + if stripped: + results.append(stripped) + return results + + async def _run_async_split(self, raw_chunks): + """Invoke the async split_chunks logic directly.""" + results = [] + + async def _gen(): + for c in raw_chunks: + yield c + + async for item in _gen(): + for chunk in item.split("\n\n"): + stripped = chunk.strip() + if stripped: + results.append(stripped) + return results + + def test_sync_single_event_per_chunk(self): + """Normal case: one SSE event per iter_text() chunk.""" + chunks = ['data: {"text":"hello"}', 'data: {"text":"world"}'] + assert self._run_sync_split(chunks) == [ + 'data: {"text":"hello"}', + 'data: {"text":"world"}', + ] + + def test_sync_multiple_events_in_one_chunk(self): + """iter_text() returns two SSE events concatenated — must be split.""" + chunks = ['data: {"text":"a"}\n\ndata: {"text":"b"}'] + assert self._run_sync_split(chunks) == [ + 'data: {"text":"a"}', + 'data: {"text":"b"}', + ] + + def test_sync_whitespace_only_chunks_discarded(self): + """Whitespace between events must not be yielded.""" + chunks = ["data: {}\n\n \n\ndata: {}"] + result = self._run_sync_split(chunks) + assert result == ["data: {}", "data: {}"] + + def test_sync_empty_string_discarded(self): + """Empty string produced by splitting trailing \\n\\n must be discarded.""" + chunks = ["data: {}\n\n"] + assert self._run_sync_split(chunks) == ["data: {}"] + + @pytest.mark.asyncio + async def test_async_whitespace_only_chunks_discarded(self): + """ + Regression test for Greptile P2: async version was checking `if not chunk` + BEFORE stripping, so '\\n ' would pass the guard and yield '' downstream, + causing ValueError in chunk_creator ('Chunk does not start with data:'). + """ + chunks = ["data: {}\n\n \n\ndata: {}"] + result = await self._run_async_split(chunks) + assert result == ["data: {}", "data: {}"] + + @pytest.mark.asyncio + async def test_async_empty_string_discarded(self): + """Trailing \\n\\n must not produce an empty yielded chunk in async path.""" + chunks = ["data: {}\n\n"] + result = await self._run_async_split(chunks) + assert result == ["data: {}"] + + @pytest.mark.asyncio + async def test_async_multiple_events_in_one_chunk(self): + """Async path must split concatenated SSE events just like sync.""" + chunks = ['data: {"text":"x"}\n\ndata: {"text":"y"}'] + result = await self._run_async_split(chunks) + assert result == ['data: {"text":"x"}', 'data: {"text":"y"}'] + + +class TestOCIProviderEmbeddingConfig: + """ + Verifies that get_provider_embedding_config returns OCIEmbedConfig for OCI + and that the dead duplicate elif branch has been removed (Greptile P1). + """ + + def test_returns_oci_embed_config(self): + from litellm.llms.oci.embed.transformation import OCIEmbedConfig + from litellm.utils import ProviderConfigManager + from litellm.types.utils import LlmProviders + + config = ProviderConfigManager.get_provider_embedding_config( + model="cohere.embed-english-v3.0", + provider=LlmProviders.OCI, + ) + assert isinstance(config, OCIEmbedConfig) + + def test_no_duplicate_oci_branch(self): + """ + Ensure utils.py does not contain two separate OCI embedding branches. + The dead code was removed in commit 64dfbe2b; this test guards against + regression (e.g. a future merge re-introducing it). + """ + import inspect + from litellm.utils import ProviderConfigManager + + source = inspect.getsource(ProviderConfigManager.get_provider_embedding_config) + oci_count = source.count("LlmProviders.OCI") + assert oci_count == 1, ( + f"Expected exactly 1 OCI branch in get_provider_embedding_config, found {oci_count}. " + "A duplicate dead-code branch may have been reintroduced." + ) + + +class TestOCICohereParamMapping: + """ + Unit tests for Bug 3 (stop → stopSequences) and Bug 4 (hardcoded defaults removed). + """ + + def _make_config(self): + return OCIChatConfig() + + def test_cohere_stop_maps_to_stop_sequences(self): + """Bug 3: Cohere API uses 'stopSequences', not 'stop'.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={"stop": ["END", "STOP"]}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + assert "stopSequences" in result, "stop should map to stopSequences for Cohere" + assert result["stopSequences"] == ["END", "STOP"] + assert "stop" not in result + + def test_generic_stop_maps_to_stop(self): + """GENERIC vendors (Meta, Google, xAI) keep 'stop' as-is.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={"stop": ["END"]}, + optional_params={}, + model="meta.llama-3.3-70b-instruct", + drop_params=False, + ) + assert result.get("stop") == ["END"] + assert "stopSequences" not in result + + def test_cohere_no_hardcoded_defaults(self): + """Bug 4: Cohere calls must not inject maxTokens/temperature/topK/topP/frequencyPenalty + when the user hasn't provided them.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + for injected in ( + "maxTokens", + "temperature", + "topK", + "topP", + "frequencyPenalty", + ): + assert ( + injected not in result + ), f"'{injected}' should not be injected when user did not provide it" + + def test_cohere_explicit_params_still_passed(self): + """User-provided Cohere params must still be forwarded correctly.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={"max_tokens": 200, "temperature": 0.5}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + assert result.get("maxTokens") == 200 + assert result.get("temperature") == 0.5 + + +class TestOCIReasoningEffort: + """ + Reasoning-effort handling for GENERIC reasoning models: + - OpenAI clients send lowercase ("low"/"medium"/"high"); OCI requires uppercase. + - OpenAI's "disable" maps to OCI's "NONE". + - Cohere on OCI has no reasoning models — the param is unsupported there. + """ + + def _build_chat_request(self, model: str, optional_params: dict) -> dict: + """Drive optional params through map → _get_optional_params and read + the resulting chatRequest body via transform_request.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + + config = OCIChatConfig() + mapped = config.map_openai_params( + non_default_params=optional_params, + optional_params={}, + model=model, + drop_params=False, + ) + body = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={**BASE_OCI_PARAMS, **mapped}, + litellm_params={}, + headers={}, + ) + return body["chatRequest"] + + def test_reasoning_effort_lowercase_uppercased(self): + chat_request = self._build_chat_request( + "xai.grok-4-fast-reasoning", + {"reasoning_effort": "low"}, + ) + assert chat_request.get("reasoningEffort") == "LOW" + + def test_reasoning_effort_disable_mapped_to_none(self): + chat_request = self._build_chat_request( + "xai.grok-4-fast-reasoning", + {"reasoning_effort": "disable"}, + ) + assert chat_request.get("reasoningEffort") == "NONE" + + def test_reasoning_effort_already_uppercase_preserved(self): + chat_request = self._build_chat_request( + "openai.gpt-5", + {"reasoning_effort": "HIGH"}, + ) + assert chat_request.get("reasoningEffort") == "HIGH" + + def test_reasoning_effort_unsupported_on_cohere_dropped(self): + """drop_params=True → silently drop reasoning_effort for Cohere.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + + config = OCIChatConfig() + result = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=True, + ) + assert "reasoning_effort" not in result + assert "reasoningEffort" not in result + + def test_reasoning_effort_unsupported_on_cohere_raises(self): + """drop_params=False → raise rather than ship a payload Cohere will reject.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + from litellm.llms.oci.common_utils import OCIError + + config = OCIChatConfig() + with pytest.raises(OCIError): + config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + + def test_reasoning_tokens_extracted_from_usage(self): + """OCI's completionTokensDetails.reasoningTokens flows into + Usage.completion_tokens_details.reasoning_tokens.""" + from litellm.llms.oci.chat.generic import handle_generic_response + + created_time = ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + oci_response = { + "modelId": "xai.grok-4-fast-reasoning", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "ok"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 5, + "completionTokens": 12, + "totalTokens": 17, + "completionTokensDetails": {"reasoningTokens": 7}, + }, + }, + } + raw = httpx.Response(status_code=200, json=oci_response) + result = handle_generic_response( + json_data=oci_response, + model="xai.grok-4-fast-reasoning", + model_response=ModelResponse(), + raw_response=raw, + ) + usage = result.usage # type: ignore[attr-defined] + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 7 + + def test_reasoning_tokens_absent_when_no_details(self): + """When OCI omits completionTokensDetails, Usage has no reasoning_tokens.""" + from litellm.llms.oci.chat.generic import handle_generic_response + + created_time = ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + oci_response = { + "modelId": "xai.grok-4", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "ok"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 5, + "completionTokens": 12, + "totalTokens": 17, + }, + }, + } + raw = httpx.Response(status_code=200, json=oci_response) + result = handle_generic_response( + json_data=oci_response, + model="xai.grok-4", + model_response=ModelResponse(), + raw_response=raw, + ) + usage = result.usage # type: ignore[attr-defined] + assert usage.completion_tokens_details is None + + +class TestOCIStreamingSignedBody: + """ + Unit test for Bug 1: sync and async streaming paths must use signed_json_body + when provided, not re-serialize data with json.dumps(). + """ + + def test_get_custom_stream_wrapper_uses_signed_body(self, monkeypatch): + """ + When signed_json_body is provided, the POST must use that exact bytes object, + not json.dumps(data) — otherwise the RSA-SHA256 signature is invalid. + """ + import httpx + from unittest.mock import MagicMock, patch + + config = OCIChatConfig() + signed_bytes = b'{"signed": true}' + posted_data = {} + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_text.return_value = iter([]) + + mock_client = MagicMock() + mock_client.post.return_value = mock_response + + def capture_post(url, **kwargs): + posted_data["data"] = kwargs.get("data") + return mock_response + + mock_client.post.side_effect = capture_post + + mock_logging = MagicMock() + + config.get_sync_custom_stream_wrapper( + api_base="https://example.com", + headers={}, + data={"key": "value"}, + messages=[], + model="meta.llama-3.3-70b-instruct", + custom_llm_provider="oci", + logging_obj=mock_logging, + client=mock_client, + signed_json_body=signed_bytes, + ) + + assert ( + posted_data["data"] == signed_bytes + ), "Streaming must use signed_json_body, not re-serialize data" + + def test_get_custom_stream_wrapper_fallback_without_signed_body(self, monkeypatch): + """When signed_json_body is None, fall back to json.dumps(data).""" + import json + from unittest.mock import MagicMock + + config = OCIChatConfig() + posted_data = {} + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_text.return_value = iter([]) + + mock_client = MagicMock() + + def capture_post(url, **kwargs): + posted_data["data"] = kwargs.get("data") + return mock_response + + mock_client.post.side_effect = capture_post + + mock_logging = MagicMock() + payload = {"key": "value"} + + config.get_sync_custom_stream_wrapper( + api_base="https://example.com", + headers={}, + data=payload, + messages=[], + model="meta.llama-3.3-70b-instruct", + custom_llm_provider="oci", + logging_obj=mock_logging, + client=mock_client, + signed_json_body=None, + ) + + assert posted_data["data"] == json.dumps( + payload + ), "Without signed_json_body, must fall back to json.dumps(data)" + + +# --------------------------------------------------------------------------- +# Additional coverage: error paths in validate_environment, transform_request, +# transform_response, and map_openai_params +# --------------------------------------------------------------------------- + + +class TestOCIChatConfigErrorPaths: + def test_validate_environment_empty_messages_raises(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="messages"): + config.validate_environment( + headers={}, + model=TEST_MODEL_NAME, + messages=[], + optional_params={ + "oci_signer": MagicMock(), + "oci_compartment_id": TEST_COMPARTMENT_ID, + }, + litellm_params={}, + ) + + def test_transform_request_missing_compartment_id_raises(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="oci_compartment_id"): + config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params={}, + litellm_params={}, + headers={}, + ) + + def test_transform_request_cohere_no_user_message_raises(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="user message"): + config.transform_request( + model="cohere.command-latest", + messages=[{"role": "system", "content": "You are helpful."}], # type: ignore + optional_params={"oci_compartment_id": TEST_COMPARTMENT_ID}, + litellm_params={}, + headers={}, + ) + + def test_transform_response_error_key_raises(self): + config = OCIChatConfig() + response = httpx.Response( + status_code=400, + json={"error": "model not found"}, + ) + with pytest.raises(Exception, match="model not found"): + config.transform_response( + model=TEST_MODEL_NAME, + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + def test_map_openai_params_unsupported_param_raises_without_drop(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="not supported on OCI"): + config.map_openai_params( + non_default_params={"audio": {"voice": "alloy"}}, + optional_params={}, + model=TEST_MODEL_NAME, + drop_params=False, + ) + + def test_map_openai_params_unsupported_param_dropped(self): + config = OCIChatConfig() + result = config.map_openai_params( + non_default_params={"audio": {"voice": "alloy"}}, + optional_params={}, + model=TEST_MODEL_NAME, + drop_params=True, + ) + assert "audio" not in result + + def test_transform_request_tool_choice_string_mapped(self): + config = OCIChatConfig() + result = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params={ + "oci_compartment_id": TEST_COMPARTMENT_ID, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "function": { + "name": "fn", + "description": "d", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + litellm_params={}, + headers={}, + ) + assert result["chatRequest"]["toolChoice"] == {"type": "AUTO"} + + +import pytest +from unittest.mock import MagicMock diff --git a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py index 388cb6224fda..cc914a22eeb4 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py @@ -5,10 +5,14 @@ from unittest.mock import patch, MagicMock from litellm import ModelResponse +from litellm.llms.oci.chat.cohere import ( + adapt_messages_to_cohere_standard, + adapt_tool_definitions_to_cohere_standard, +) from litellm.llms.oci.chat.transformation import ( OCIChatConfig, - get_vendor_from_model, OCIStreamWrapper, + get_vendor_from_model, ) from litellm.types.llms.oci import OCIVendors @@ -75,7 +79,7 @@ def test_cohere_tool_definition_transformation(self): ] # Transform tools - cohere_tools = config.adapt_tool_definitions_to_cohere_standard(openai_tools) + cohere_tools = adapt_tool_definitions_to_cohere_standard(openai_tools) # Verify transformation assert len(cohere_tools) == 2 @@ -90,13 +94,16 @@ def test_cohere_tool_definition_transformation(self): # Check location parameter location_param = weather_tool.parameterDefinitions["location"] assert location_param.description == "The city or location to get weather for" - assert location_param.type == "string" + assert location_param.type == "str" assert location_param.isRequired == True # Check unit parameter unit_param = weather_tool.parameterDefinitions["unit"] - assert unit_param.description == "Temperature unit (celsius or fahrenheit)" - assert unit_param.type == "string" + assert ( + unit_param.description + == "Temperature unit (celsius or fahrenheit). Allowed values: ['celsius', 'fahrenheit']" + ) + assert unit_param.type == "str" assert unit_param.isRequired == False # Check second tool @@ -107,7 +114,7 @@ def test_cohere_tool_definition_transformation(self): expression_param = calc_tool.parameterDefinitions["expression"] assert expression_param.description == "Mathematical expression to evaluate" - assert expression_param.type == "string" + assert expression_param.type == "str" assert expression_param.isRequired == True def test_cohere_request_with_tools(self): @@ -157,13 +164,6 @@ def test_cohere_request_with_tools(self): assert chat_request["message"] == "What's the weather like in Tokyo?" assert chat_request["chatHistory"] == [] - # Verify default parameters are included - assert chat_request["maxTokens"] == 600 - assert chat_request["temperature"] == 1 - assert chat_request["topK"] == 0 - assert chat_request["topP"] == 0.75 - assert chat_request["frequencyPenalty"] == 0 - # Verify tools are transformed correctly assert "tools" in chat_request assert len(chat_request["tools"]) == 1 @@ -226,7 +226,7 @@ def test_cohere_response_with_tool_calls(self): assert len(result.choices[0].message.tool_calls) == 1 tool_call = result.choices[0].message.tool_calls[0] - assert tool_call.id == "call_0" + assert tool_call.id.startswith("call_") assert tool_call.type == "function" assert tool_call.function.name == "get_weather" assert tool_call.function.arguments == '{"location": "Tokyo"}' @@ -324,22 +324,26 @@ def test_cohere_tool_call_only_message_no_text(self): }, ] - chat_history = config.adapt_messages_to_cohere_standard(messages) + chat_history = adapt_messages_to_cohere_standard(messages) - # First message is the user message - assert chat_history[0].role == "USER" - assert chat_history[0].message == "What's the weather?" + # The last user message is consumed by the request's top-level `message` + # field, so chatHistory carries the assistant tool call and tool result. + assert len(chat_history) == 2 - # Second message is the assistant with tool calls and no text - assistant_msg = chat_history[1] + assistant_msg = chat_history[0] assert assistant_msg.role == "CHATBOT" assert assistant_msg.message is None or assistant_msg.message == "" assert assistant_msg.toolCalls is not None assert len(assistant_msg.toolCalls) == 1 assert assistant_msg.toolCalls[0].name == "get_weather" + tool_msg = chat_history[1] + assert tool_msg.role == "TOOL" + assert tool_msg.toolResults[0].call.name == "get_weather" + assert tool_msg.toolResults[0].outputs[0]["output"] == "Sunny, 25C" + def test_cohere_chat_history_with_tool_calls(self): - """Test chat history transformation with tool calls""" + """Tool results trailing the last user turn must be preserved in chatHistory.""" config = OCIChatConfig() messages = [ @@ -365,28 +369,29 @@ def test_cohere_chat_history_with_tool_calls(self): }, ] - chat_history = config.adapt_messages_to_cohere_standard(messages) + chat_history = adapt_messages_to_cohere_standard(messages) - # Verify chat history structure (excludes last message) + # The last user message becomes the request's top-level `message`. + # Everything else — including the trailing tool result — must remain in + # chatHistory so the model can see the tool output. assert len(chat_history) == 2 - # Check user message - user_msg = chat_history[0] - assert user_msg.role == "USER" - assert user_msg.message == "What's the weather like in Tokyo?" - - # Check assistant message with tool calls - assistant_msg = chat_history[1] + assistant_msg = chat_history[0] assert assistant_msg.role == "CHATBOT" assert assistant_msg.message == "I will look up the weather in Tokyo." assert assistant_msg.toolCalls is not None assert len(assistant_msg.toolCalls) == 1 assert assistant_msg.toolCalls[0].name == "get_weather" - # The parameters should be parsed as JSON assert assistant_msg.toolCalls[0].parameters == {"location": "Tokyo"} - # Note: The tool message (last message) is excluded from chat history - # This is the expected behavior for Cohere models + tool_msg = chat_history[1] + assert tool_msg.role == "TOOL" + assert tool_msg.toolResults[0].call.name == "get_weather" + assert tool_msg.toolResults[0].call.parameters == {"location": "Tokyo"} + assert ( + tool_msg.toolResults[0].outputs[0]["output"] + == "The weather in Tokyo is 22°C with partly cloudy skies." + ) def test_cohere_streaming_chunk_handling(self): """Test Cohere streaming chunk handling""" @@ -457,7 +462,7 @@ def test_cohere_parameter_mapping_excludes_tool_choice(self): assert "tool_choice" not in supported_params def test_cohere_default_parameters(self): - """Test that Cohere requests include required default parameters""" + """Test that Cohere requests do not inject hardcoded defaults — caller supplies all params.""" config = OCIChatConfig() messages = [{"role": "user", "content": "Hello"}] optional_params = {"oci_compartment_id": TEST_COMPARTMENT_ID} @@ -472,12 +477,11 @@ def test_cohere_default_parameters(self): chat_request = transformed_request["chatRequest"] - # Verify all required default parameters are present - assert chat_request["maxTokens"] == 600 - assert chat_request["temperature"] == 1 - assert chat_request["topK"] == 0 - assert chat_request["topP"] == 0.75 - assert chat_request["frequencyPenalty"] == 0 + # No hardcoded defaults injected — only pass through what the user supplies + assert "maxTokens" not in chat_request + assert "topK" not in chat_request + assert "topP" not in chat_request + assert "frequencyPenalty" not in chat_request def test_cohere_parameter_override(self): """Test that user-provided parameters override defaults""" @@ -499,14 +503,104 @@ def test_cohere_parameter_override(self): chat_request = transformed_request["chatRequest"] - # Verify user parameters override defaults + # Verify user parameters are passed through assert chat_request["temperature"] == 0.5 assert chat_request["maxTokens"] == 1000 - # Verify other defaults are still present - assert chat_request["topK"] == 0 - assert chat_request["topP"] == 0.75 - assert chat_request["frequencyPenalty"] == 0 + # Unset params are absent (no hardcoded defaults) + assert "topK" not in chat_request + assert "topP" not in chat_request + assert "frequencyPenalty" not in chat_request + + def test_cohere_response_finish_reason_tool_call(self): + """Test that finishReason='TOOL_CALL' is accepted by Pydantic and mapped to 'tool_calls'.""" + config = OCIChatConfig() + + mock_cohere_response = { + "modelId": "cohere.command-latest", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "COHERE", + "text": "", + "finishReason": "TOOL_CALL", + "toolCalls": [ + {"name": "get_weather", "parameters": {"location": "London"}} + ], + "usage": { + "promptTokens": 20, + "completionTokens": 10, + "totalTokens": 30, + }, + }, + } + + response = httpx.Response( + status_code=200, + json=mock_cohere_response, + headers={"Content-Type": "application/json"}, + ) + + result = config.transform_response( + model="cohere.command-latest", + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].finish_reason == "tool_calls" + assert result.choices[0].message.tool_calls is not None + assert len(result.choices[0].message.tool_calls) == 1 + assert result.choices[0].message.tool_calls[0].function.name == "get_weather" + + def test_cohere_response_unknown_finish_reason_degrades_to_stop(self): + """A future/unknown finishReason in non-streaming responses must + degrade to ``stop`` via ``handle_cohere_response``'s fallback + rather than crash Pydantic validation. Mirrors the streaming + handler's behavior. See bug caf74429. + """ + config = OCIChatConfig() + + mock_cohere_response = { + "modelId": "cohere.command-latest", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "COHERE", + "text": "hello", + "finishReason": "FUTURE_REASON_NOT_YET_KNOWN", + "usage": { + "promptTokens": 1, + "completionTokens": 1, + "totalTokens": 2, + }, + }, + } + + response = httpx.Response( + status_code=200, + json=mock_cohere_response, + headers={"Content-Type": "application/json"}, + ) + + result = config.transform_response( + model="cohere.command-latest", + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].finish_reason == "stop" def test_cohere_vendor_detection(self): """Test that Cohere models are correctly identified""" @@ -532,7 +626,7 @@ def test_cohere_error_handling_invalid_tool_format(self): ] # The function should handle missing function key gracefully - cohere_tools = config.adapt_tool_definitions_to_cohere_standard(invalid_tools) + cohere_tools = adapt_tool_definitions_to_cohere_standard(invalid_tools) # Should create a tool with empty name and description assert len(cohere_tools) == 1 @@ -686,16 +780,114 @@ def test_system_messages_excluded_from_chat_history(self): {"role": "assistant", "content": "First answer"}, {"role": "user", "content": "Second question"}, ] + optional_params = {"oci_compartment_id": TEST_COMPARTMENT_ID} - chat_history = config.adapt_messages_to_cohere_standard(messages) + result = config.transform_request( + model="cohere.command-latest", + messages=messages, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) - # Should contain user and assistant only, no system - # Note: adapt_messages_to_cohere_standard excludes the last message - roles = [msg.role for msg in chat_history] + chat_request = result["chatRequest"] + roles = [msg["role"] for msg in chat_request["chatHistory"]] assert "SYSTEM" not in roles assert roles == ["USER", "CHATBOT"] +class TestCohereStreamChunkEdgeCases: + """Additional coverage for handle_cohere_stream_chunk error/edge paths.""" + + def _wrapper(self): + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + return OCIStreamWrapper( + completion_stream=MagicMock(), + model="cohere.command-latest", + logging_obj=MagicMock(), + ) + + def test_stream_chunk_tool_call_finish_reason(self): + wrapper = self._wrapper() + chunk = { + "apiFormat": "COHERE", + "text": "", + "index": 0, + "finishReason": "TOOL_CALL", + } + result = wrapper.chunk_creator(f"data: {json.dumps(chunk)}") + assert result.choices[0].finish_reason == "tool_calls" + + def test_stream_chunk_max_tokens_finish_reason(self): + wrapper = self._wrapper() + chunk = { + "apiFormat": "COHERE", + "text": "truncated", + "index": 0, + "finishReason": "MAX_TOKENS", + } + result = wrapper.chunk_creator(f"data: {json.dumps(chunk)}") + assert result.choices[0].finish_reason == "length" + + def test_stream_chunk_unknown_finish_reason_does_not_raise(self): + from litellm.llms.oci.chat.cohere import handle_cohere_stream_chunk + + chunk = { + "apiFormat": "COHERE", + "text": "", + "index": 0, + "finishReason": "FUTURE_REASON", + } + # Should not raise — unknown reasons fall through the elif chain unchanged + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0] is not None + + def test_stream_chunk_null_index_defaults_to_zero(self): + wrapper = self._wrapper() + chunk = {"apiFormat": "COHERE", "text": "hi", "index": None} + result = wrapper.chunk_creator(f"data: {json.dumps(chunk)}") + assert result.choices[0].index == 0 + + +class TestCohereMessageAdaptationEdgeCases: + """Coverage for adapt_messages_to_cohere_standard error paths.""" + + def test_json_decode_error_in_tool_args_defaults_to_empty(self): + from litellm.llms.oci.chat.cohere import adapt_messages_to_cohere_standard + + messages = [ + { + "role": "assistant", + "content": "calling", + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "fn", "arguments": "NOT JSON {{{"}, + } + ], + }, + {"role": "user", "content": "follow up"}, + ] + # Should not raise — bad JSON defaults to empty params {} + history = adapt_messages_to_cohere_standard(messages) + assert history[0].toolCalls[0].parameters == {} + + def test_extract_text_content_list_with_non_dict_items(self): + from litellm.llms.oci.chat.cohere import _extract_text_content + + # List with a non-dict item — should be silently skipped + result = _extract_text_content([{"type": "text", "text": "hello"}, "bad_item"]) + assert result == "hello" + + def test_extract_text_content_non_string_non_list(self): + from litellm.llms.oci.chat.cohere import _extract_text_content + + result = _extract_text_content(12345) + assert result == "12345" + + class TestOCICohereStreaming: """Test Cohere streaming functionality""" @@ -713,9 +905,9 @@ def test_cohere_streaming_wrapper_initialization(self): """Test OCIStreamWrapper initialization""" stream_wrapper = self._create_stream_wrapper() + # chunk_creator is the public dispatch entry point assert hasattr(stream_wrapper, "chunk_creator") - assert hasattr(stream_wrapper, "_handle_cohere_stream_chunk") - assert hasattr(stream_wrapper, "_handle_generic_stream_chunk") + assert callable(stream_wrapper.chunk_creator) def test_cohere_streaming_chunk_parsing(self): """Test parsing of Cohere streaming chunks""" @@ -739,10 +931,12 @@ def test_cohere_streaming_invalid_chunk_format(self): def test_cohere_streaming_non_json_chunk(self): """Test error handling for non-JSON chunk""" + from litellm.llms.oci.common_utils import OCIError + stream_wrapper = self._create_stream_wrapper() # Test non-JSON chunk - with pytest.raises(json.JSONDecodeError): + with pytest.raises(OCIError, match="Chunk cannot be parsed as JSON"): stream_wrapper.chunk_creator("data: invalid json") def test_cohere_streaming_generic_chunk_fallback(self): diff --git a/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py b/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py new file mode 100644 index 000000000000..7583e3bc1835 --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py @@ -0,0 +1,440 @@ +""" +Unit tests for litellm/llms/oci/chat/generic.py — error paths and stream handling. +""" + +import json +import pytest +from unittest.mock import MagicMock + +import httpx + +from litellm import ModelResponse +from litellm.llms.oci.chat.generic import ( + adapt_messages_to_generic_oci_standard, + adapt_messages_to_generic_oci_standard_content_message, + adapt_messages_to_generic_oci_standard_tool_call, + handle_generic_response, + handle_generic_stream_chunk, +) +from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIStreamWrapper +from litellm.llms.oci.common_utils import OCIError + +# --------------------------------------------------------------------------- +# adapt_messages_to_generic_oci_standard_content_message — error paths +# --------------------------------------------------------------------------- + + +class TestGenericContentMessageErrors: + def test_non_dict_content_item_raises(self): + with pytest.raises(OCIError, match="must be a dictionary"): + adapt_messages_to_generic_oci_standard_content_message( + "user", ["not a dict"] + ) + + def test_non_string_type_field_raises(self): + with pytest.raises(OCIError, match="string `type` field"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": 123, "text": "hi"}] + ) + + def test_unsupported_content_type_raises(self): + with pytest.raises(OCIError, match="not supported by OCI"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "video_url", "url": "https://example.com/v.mp4"}] + ) + + def test_non_string_text_raises(self): + with pytest.raises(OCIError, match="must have a string `text` field"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "text", "text": 42}] + ) + + def test_image_url_as_invalid_type_raises(self): + with pytest.raises(OCIError, match="must be a string or an object"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "image_url", "image_url": 99}] + ) + + def test_image_url_as_string(self): + msg = adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "image_url", "image_url": "https://example.com/img.png"}] + ) + assert msg.content[0].imageUrl.url == "https://example.com/img.png" + + def test_image_url_as_dict(self): + msg = adapt_messages_to_generic_oci_standard_content_message( + "user", + [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/img.png"}, + } + ], + ) + assert msg.content[0].imageUrl.url == "https://example.com/img.png" + + def test_text_content_string(self): + msg = adapt_messages_to_generic_oci_standard_content_message("user", "hello") + assert msg.content[0].text == "hello" + + +# --------------------------------------------------------------------------- +# adapt_messages_to_generic_oci_standard_tool_call — error paths +# --------------------------------------------------------------------------- + + +class TestGenericToolCallErrors: + def test_non_dict_tool_call_raises(self): + with pytest.raises(OCIError, match="must be a dictionary"): + adapt_messages_to_generic_oci_standard_tool_call("assistant", ["bad"]) + + def test_non_function_type_raises(self): + with pytest.raises(OCIError, match="only supports function tool calls"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "database", + "id": "x", + "function": {"name": "f", "arguments": "{}"}, + } + ], + ) + + def test_non_string_id_raises(self): + with pytest.raises(OCIError, match="id.*must be a string"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "function", + "id": 123, + "function": {"name": "f", "arguments": "{}"}, + } + ], + ) + + def test_non_dict_function_raises(self): + with pytest.raises(OCIError, match="`function` must be a dictionary"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [{"type": "function", "id": "c1", "function": "not_a_dict"}], + ) + + def test_non_string_function_name_raises(self): + with pytest.raises(OCIError, match="function.name.*must be a string"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "function", + "id": "c1", + "function": {"name": 5, "arguments": "{}"}, + } + ], + ) + + def test_non_string_arguments_raises(self): + with pytest.raises(OCIError, match="arguments.*must be a JSON string"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "function", + "id": "c1", + "function": {"name": "fn", "arguments": {"key": "val"}}, + } + ], + ) + + +# --------------------------------------------------------------------------- +# adapt_messages_to_generic_oci_standard — combined paths +# --------------------------------------------------------------------------- + + +class TestGenericMessageAdaptation: + def test_tool_calls_not_list_raises(self): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": "not_a_list", + } + ] + with pytest.raises(OCIError, match="`tool_calls` must be a list"): + adapt_messages_to_generic_oci_standard(messages) + + def test_tool_result_non_string_tool_call_id_raises(self): + messages = [{"role": "tool", "content": "result", "tool_call_id": 999}] + with pytest.raises(OCIError, match="string `tool_call_id`"): + adapt_messages_to_generic_oci_standard(messages) + + def test_tool_result_non_string_content_raises(self): + messages = [ + {"role": "tool", "content": {"structured": "data"}, "tool_call_id": "c1"} + ] + with pytest.raises(OCIError, match="`content` must be a string"): + adapt_messages_to_generic_oci_standard(messages) + + def test_non_string_non_list_content_raises(self): + messages = [{"role": "user", "content": 42}] + with pytest.raises(OCIError, match="`content` must be a string or list"): + adapt_messages_to_generic_oci_standard(messages) + + +# --------------------------------------------------------------------------- +# handle_generic_response — error and None message paths +# --------------------------------------------------------------------------- + + +class TestHandleGenericResponse: + def _make_response(self, body: dict, status: int = 200) -> httpx.Response: + return httpx.Response(status_code=status, json=body) + + def _valid_body(self, message=None): + return { + "modelId": "xai.grok-4", + "modelVersion": "1", + "chatResponse": { + "apiFormat": "GENERIC", + "timeCreated": "2024-01-01T00:00:00Z", + "choices": [ + {"message": message, "finishReason": "COMPLETE", "index": 0} + ], + "usage": {"promptTokens": 5, "completionTokens": 5, "totalTokens": 10}, + }, + } + + def test_none_response_message(self): + body = self._valid_body(message=None) + raw = self._make_response(body) + # Should not raise — None message means no content set + result = handle_generic_response(body, "xai.grok-4", ModelResponse(), raw) + assert result.model == "xai.grok-4" + + def test_response_with_text_content(self): + body = self._valid_body( + message={ + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "Hello!"}], + } + ) + raw = self._make_response(body) + result = handle_generic_response(body, "xai.grok-4", ModelResponse(), raw) + assert result.choices[0].message.content == "Hello!" + + def test_response_with_tool_calls(self): + body = self._valid_body( + message={ + "role": "ASSISTANT", + "content": [], + "toolCalls": [ + { + "id": "call_abc", + "type": "FUNCTION", + "name": "get_weather", + "arguments": '{"location": "Tokyo"}', + } + ], + } + ) + raw = self._make_response(body) + result = handle_generic_response(body, "xai.grok-4", ModelResponse(), raw) + assert result.choices[0].message.tool_calls is not None + + +# --------------------------------------------------------------------------- +# handle_generic_stream_chunk — finish reasons and error paths +# --------------------------------------------------------------------------- + + +class TestHandleGenericStreamChunk: + def test_max_tokens_finish_reason(self): + chunk = {"apiFormat": "GENERIC", "index": 0, "finishReason": "MAX_TOKENS"} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "length" + + def test_tool_calls_finish_reason(self): + chunk = {"apiFormat": "GENERIC", "index": 0, "finishReason": "TOOL_CALLS"} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "tool_calls" + + def test_unknown_finish_reason_does_not_raise(self): + chunk = {"apiFormat": "GENERIC", "index": 0, "finishReason": "SOME_NEW_REASON"} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0] is not None + + def test_null_index_defaults_to_zero(self): + chunk = {"apiFormat": "GENERIC", "index": None, "finishReason": None} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].index == 0 + + def test_image_content_in_stream_raises(self): + from litellm.types.llms.oci import OCIImageContentPart, OCIImageUrl, OCIMessage + + chunk = { + "apiFormat": "GENERIC", + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [ + { + "type": "IMAGE", + "imageUrl": {"url": "https://example.com/img.png"}, + } + ], + }, + } + with pytest.raises(OCIError, match="image content"): + handle_generic_stream_chunk(chunk) + + def test_stream_chunk_with_tool_calls(self): + chunk = { + "apiFormat": "GENERIC", + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [], + "toolCalls": [ + { + "id": "call_abc", + "type": "FUNCTION", + "name": "get_weather", + "arguments": '{"location": "Tokyo"}', + } + ], + }, + } + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].delta.tool_calls is not None + + +# --------------------------------------------------------------------------- +# OCIStreamWrapper.chunk_creator — non-string chunk +# --------------------------------------------------------------------------- + + +class TestOCIStreamWrapperChunkCreator: + def _wrapper(self): + return OCIStreamWrapper( + completion_stream=MagicMock(), + model="xai.grok-4", + logging_obj=MagicMock(), + ) + + def test_non_string_chunk_raises(self): + w = self._wrapper() + with pytest.raises(ValueError, match="not a string"): + w.chunk_creator({"already": "parsed"}) + + +# --------------------------------------------------------------------------- +# GPT-5 family: maxCompletionTokens routing +# +# Regression guard: OCI rejects "maxTokens" for openai.gpt-5* models with HTTP +# 400 ("Use 'maxCompletionTokens' instead.") — verified against live OCI. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def _register_oci_gpt5_in_catalog(): + """Guarantee OCI GPT-5 catalog entries with supports_reasoning=True are + present for the duration of the test, regardless of whether + ``litellm.model_cost`` was populated from the bundled + ``model_prices_and_context_window.json`` (which ships them) or from a + remote map that may lag behind. + """ + import litellm + + needed = { + "oci/openai.gpt-5", + "oci/openai.gpt-5-mini", + "oci/openai.gpt-5-nano", + } + added = [] + for key in needed: + if key not in litellm.model_cost: + litellm.model_cost[key] = { + "litellm_provider": "oci", + "mode": "chat", + "supports_reasoning": True, + } + added.append(key) + yield + for key in added: + litellm.model_cost.pop(key, None) + + +class TestGpt5MaxCompletionTokens: + def test_helper_detects_gpt5_family(self, _register_oci_gpt5_in_catalog): + from litellm.llms.oci.chat.transformation import ( + _model_uses_max_completion_tokens, + ) + + assert _model_uses_max_completion_tokens("openai.gpt-5") is True + assert _model_uses_max_completion_tokens("openai.gpt-5-mini") is True + assert _model_uses_max_completion_tokens("openai.gpt-5-nano") is True + assert _model_uses_max_completion_tokens("oci/openai.gpt-5") is True + + assert _model_uses_max_completion_tokens("openai.gpt-oss-120b") is False + assert _model_uses_max_completion_tokens("meta.llama-3.3-70b-instruct") is False + assert _model_uses_max_completion_tokens("cohere.command-latest") is False + assert _model_uses_max_completion_tokens("") is False + + def test_gpt5_routes_max_tokens_to_max_completion_tokens( + self, _register_oci_gpt5_in_catalog + ): + from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors + + cfg = OCIChatConfig() + # Both shapes optional_params can take after upstream map_openai_params: + # 1. openai-side key still present + out_a = cfg._get_optional_params( + OCIVendors.GENERIC, {"max_tokens": 64}, model="openai.gpt-5" + ) + assert out_a.get("maxCompletionTokens") == 64 + assert "maxTokens" not in out_a + + # 2. already pre-translated to OCI alias + out_b = cfg._get_optional_params( + OCIVendors.GENERIC, {"maxTokens": 64}, model="openai.gpt-5-mini" + ) + assert out_b.get("maxCompletionTokens") == 64 + assert "maxTokens" not in out_b + + def test_non_gpt5_keeps_max_tokens(self): + from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors + + cfg = OCIChatConfig() + out = cfg._get_optional_params( + OCIVendors.GENERIC, + {"max_tokens": 64}, + model="meta.llama-3.3-70b-instruct", + ) + assert out.get("maxTokens") == 64 + assert "maxCompletionTokens" not in out + + def test_cohere_reasoning_model_keeps_max_tokens(self): + from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors + + cfg = OCIChatConfig() + out = cfg._get_optional_params( + OCIVendors.COHERE, + {"max_tokens": 64}, + model="cohere.command-a-reasoning", + ) + assert out.get("maxTokens") == 64 + assert "maxCompletionTokens" not in out + + def test_payload_serializes_max_completion_tokens(self): + from litellm.types.llms.oci import OCIChatRequestPayload + + payload = OCIChatRequestPayload( + apiFormat="GENERIC", + messages=[], + maxCompletionTokens=64, + ) + dumped = payload.model_dump(exclude_none=True) + assert dumped["maxCompletionTokens"] == 64 + assert "maxTokens" not in dumped diff --git a/tests/test_litellm/llms/oci/chat/test_oci_sse_splitter.py b/tests/test_litellm/llms/oci/chat/test_oci_sse_splitter.py new file mode 100644 index 000000000000..a2faf664ee76 --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_oci_sse_splitter.py @@ -0,0 +1,232 @@ +""" +Tests for the OCI SSE event splitter. + +Regression coverage for the streaming bug John Lathouwers reported: the old +``split_chunks`` helper split each individual HTTP read on ``\\n\\n``, so any +event that straddled a read boundary or any pair of events separated by a +single ``\\n`` would yield malformed chunks to ``OCIStreamWrapper.chunk_creator`` +and crash ``json.loads``. +""" + +import asyncio +from typing import AsyncIterator, Iterator, List + +from litellm.llms.oci.chat.transformation import ( + _aiter_sse_events, + _iter_sse_events, +) + + +def _collect_sync(stream: Iterator[str]) -> List[str]: + return list(_iter_sse_events(iter(stream))) + + +def _collect_async(chunks: List[str]) -> List[str]: + async def _src() -> AsyncIterator[str]: + for c in chunks: + yield c + + async def _run() -> List[str]: + out: List[str] = [] + async for line in _aiter_sse_events(_src()): + out.append(line) + return out + + return asyncio.run(_run()) + + +# --------------------------------------------------------------------------- +# Sync splitter +# --------------------------------------------------------------------------- + + +class TestIterSseEventsSync: + def test_well_formed_double_newline_separators(self): + reads = ['data: {"a":1}\n\ndata: {"a":2}\n\n'] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_event_split_across_two_reads(self): + # The bug: read 1 ends mid-JSON, read 2 finishes it. Old code would + # have yielded a truncated 'data: {"index":0,"text":"hel' and crashed + # json.loads in chunk_creator. + reads = [ + 'data: {"index":0,"text":"hel', + 'lo"}\n\n', + ] + assert _collect_sync(reads) == ['data: {"index":0,"text":"hello"}'] + + def test_event_split_into_many_tiny_reads(self): + full = 'data: {"k":"value with spaces"}\n\n' + reads = [full[i : i + 3] for i in range(0, len(full), 3)] + assert _collect_sync(reads) == ['data: {"k":"value with spaces"}'] + + def test_single_newline_separator(self): + # The other shape John saw: events separated by just '\n'. + reads = ['data: {"a":1}\ndata: {"a":2}\ndata: {"a":3}\n'] + assert _collect_sync(reads) == [ + 'data: {"a":1}', + 'data: {"a":2}', + 'data: {"a":3}', + ] + + def test_mixed_separators_in_one_read(self): + reads = ['data: {"a":1}\ndata: {"a":2}\n\ndata: {"a":3}\n\n'] + assert _collect_sync(reads) == [ + 'data: {"a":1}', + 'data: {"a":2}', + 'data: {"a":3}', + ] + + def test_keepalive_and_comment_lines_dropped(self): + # SSE keepalives ("\n") and comment lines (": ping") must not be + # forwarded to chunk_creator, which would reject anything not + # starting with 'data:'. + reads = [ + "\n", + ": ping\n", + 'data: {"a":1}\n\n', + "\n", + ": keepalive\n\n", + 'data: {"a":2}\n\n', + ] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_trailing_partial_event_flushed_at_eof(self): + # Final event arrives without a terminating newline. The splitter + # must still emit it once the upstream iterator is exhausted. + reads = ['data: {"a":1}\n\n', 'data: {"a":2}'] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_trailing_non_data_line_dropped_at_eof(self): + reads = ['data: {"a":1}\n\n: trailing-comment'] + assert _collect_sync(reads) == ['data: {"a":1}'] + + def test_empty_stream(self): + assert _collect_sync([]) == [] + + def test_only_whitespace_and_keepalives(self): + assert _collect_sync(["\n", "\n\n", ": ping\n"]) == [] + + def test_boundary_between_data_keyword_and_payload(self): + # The 'data:' marker itself straddles a read boundary. + reads = ["dat", 'a: {"a":1}\n\n'] + assert _collect_sync(reads) == ['data: {"a":1}'] + + def test_carriage_return_in_payload_preserved(self): + # SSE-over-the-wire may use \r\n line endings. We split on \n; the + # \r ends up on the previous line and strip() removes it. + reads = ['data: {"a":1}\r\ndata: {"a":2}\r\n'] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + +# --------------------------------------------------------------------------- +# Async splitter — same scenarios, parallel coverage +# --------------------------------------------------------------------------- + + +class TestIterSseEventsAsync: + def test_well_formed_double_newline_separators(self): + assert _collect_async(['data: {"a":1}\n\ndata: {"a":2}\n\n']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + ] + + def test_event_split_across_two_reads(self): + assert _collect_async(['data: {"index":0,"text":"hel', 'lo"}\n\n']) == [ + 'data: {"index":0,"text":"hello"}' + ] + + def test_event_split_into_many_tiny_reads(self): + full = 'data: {"k":"value with spaces"}\n\n' + reads = [full[i : i + 3] for i in range(0, len(full), 3)] + assert _collect_async(reads) == ['data: {"k":"value with spaces"}'] + + def test_single_newline_separator(self): + assert _collect_async(['data: {"a":1}\ndata: {"a":2}\ndata: {"a":3}\n']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + 'data: {"a":3}', + ] + + def test_mixed_separators_in_one_read(self): + assert _collect_async( + ['data: {"a":1}\ndata: {"a":2}\n\ndata: {"a":3}\n\n'] + ) == ['data: {"a":1}', 'data: {"a":2}', 'data: {"a":3}'] + + def test_keepalive_and_comment_lines_dropped(self): + reads = [ + "\n", + ": ping\n", + 'data: {"a":1}\n\n', + "\n", + ": keepalive\n\n", + 'data: {"a":2}\n\n', + ] + assert _collect_async(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_trailing_partial_event_flushed_at_eof(self): + assert _collect_async(['data: {"a":1}\n\n', 'data: {"a":2}']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + ] + + def test_trailing_non_data_line_dropped_at_eof(self): + assert _collect_async(['data: {"a":1}\n\n: trailing-comment']) == [ + 'data: {"a":1}' + ] + + def test_empty_stream(self): + assert _collect_async([]) == [] + + def test_only_whitespace_and_keepalives(self): + assert _collect_async(["\n", "\n\n", ": ping\n"]) == [] + + def test_boundary_between_data_keyword_and_payload(self): + assert _collect_async(["dat", 'a: {"a":1}\n\n']) == ['data: {"a":1}'] + + def test_carriage_return_in_payload_preserved(self): + assert _collect_async(['data: {"a":1}\r\ndata: {"a":2}\r\n']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + ] + + +# --------------------------------------------------------------------------- +# End-to-end: feed an awkwardly-chunked stream into OCIStreamWrapper and +# verify chunk_creator still parses each yielded line. This is the smoke +# test that proves the integration with the downstream consumer holds. +# --------------------------------------------------------------------------- + + +class TestSseSplitterFeedsChunkCreator: + def test_split_event_parses_cleanly(self): + # Build a realistic GENERIC OCI streaming payload, then chop it into + # awkward reads. The splitter must reassemble exactly one event so + # json.loads inside chunk_creator does not raise. + import json + from unittest.mock import MagicMock + + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + payload = { + "apiFormat": "GENERIC", + "message": {"content": [{"text": "hello"}]}, + "finishReason": None, + } + wire = f"data: {json.dumps(payload)}\n\n" + # Split the wire string at an awkward point inside the JSON body. + cut = wire.index('"hello"') + 3 + reads = [wire[:cut], wire[cut:]] + + # Drive the splitter directly and confirm we get exactly one event. + events = list(_iter_sse_events(iter(reads))) + assert len(events) == 1 + assert events[0].startswith("data: ") + # chunk_creator should now parse this without raising. + wrapper = OCIStreamWrapper( + completion_stream=MagicMock(), + model="xai.grok-4", + logging_obj=MagicMock(), + ) + # Must not raise. + wrapper.chunk_creator(events[0]) diff --git a/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py index f9d4be8032a0..acad5da93e20 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py @@ -11,13 +11,10 @@ import os import sys -import pytest -from unittest.mock import MagicMock -# Adds the parent directory to the system path sys.path.insert(0, os.path.abspath("../../../../..")) -from litellm.llms.oci.chat.transformation import OCIStreamWrapper +from litellm.llms.oci.chat.generic import handle_generic_stream_chunk from litellm.types.utils import ModelResponseStream @@ -26,12 +23,9 @@ class TestOCIStreamingToolCalls: def test_stream_chunk_with_missing_arguments_field(self): """ - Test that streaming chunks with tool calls missing 'arguments' field are handled. - OCI API can return tool calls in early chunks without the 'arguments' field, which should be filled with an empty string to satisfy Pydantic validation. """ - # Mock streaming chunk with tool call missing 'arguments' field chunk_data = { "index": 0, "finishReason": None, @@ -43,21 +37,13 @@ def test_stream_chunk_with_missing_arguments_field(self): "type": "FUNCTION", "id": "call_abc123", "name": "get_weather", - # Note: 'arguments' field is missing + # 'arguments' field is missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - # This should not raise a ValidationError - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert len(result.choices) == 1 @@ -66,9 +52,7 @@ def test_stream_chunk_with_missing_arguments_field(self): assert result.choices[0].delta.tool_calls[0]["function"]["arguments"] == "" def test_stream_chunk_with_missing_id_field(self): - """ - Test that streaming chunks with tool calls missing 'id' field are handled. - """ + """Missing 'id' gets a generated call_* id.""" chunk_data = { "index": 0, "finishReason": None, @@ -80,29 +64,20 @@ def test_stream_chunk_with_missing_id_field(self): "type": "FUNCTION", "name": "get_weather", "arguments": '{"location": "San Francisco"}', - # Note: 'id' field is missing + # 'id' field is missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None - assert result.choices[0].delta.tool_calls[0]["id"] == "" + assert result.choices[0].delta.tool_calls[0]["id"].startswith("call_") def test_stream_chunk_with_missing_name_field(self): - """ - Test that streaming chunks with tool calls missing 'name' field are handled. - """ + """Missing 'name' defaults to empty string.""" chunk_data = { "index": 0, "finishReason": None, @@ -114,29 +89,20 @@ def test_stream_chunk_with_missing_name_field(self): "type": "FUNCTION", "id": "call_abc123", "arguments": '{"location": "San Francisco"}', - # Note: 'name' field is missing + # 'name' field is missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None assert result.choices[0].delta.tool_calls[0]["function"]["name"] == "" def test_stream_chunk_with_all_missing_fields(self): - """ - Test that streaming chunks with tool calls missing all optional fields are handled. - """ + """All optional fields missing — all default gracefully.""" chunk_data = { "index": 0, "finishReason": None, @@ -146,31 +112,22 @@ def test_stream_chunk_with_all_missing_fields(self): "toolCalls": [ { "type": "FUNCTION" - # All fields missing: id, name, arguments + # id, name, arguments all missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None - assert result.choices[0].delta.tool_calls[0]["id"] == "" + assert result.choices[0].delta.tool_calls[0]["id"].startswith("call_") assert result.choices[0].delta.tool_calls[0]["function"]["name"] == "" assert result.choices[0].delta.tool_calls[0]["function"]["arguments"] == "" def test_stream_chunk_with_complete_tool_call(self): - """ - Test that streaming chunks with complete tool calls still work correctly. - """ + """Fully-populated tool call passes through unchanged.""" chunk_data = { "index": 0, "finishReason": None, @@ -188,14 +145,7 @@ def test_stream_chunk_with_complete_tool_call(self): }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None @@ -210,9 +160,7 @@ def test_stream_chunk_with_complete_tool_call(self): ) def test_stream_chunk_with_multiple_tool_calls_missing_fields(self): - """ - Test that streaming chunks with multiple tool calls, some with missing fields, are handled. - """ + """Multiple tool calls with a mix of complete and incomplete entries.""" chunk_data = { "index": 0, "finishReason": None, @@ -220,58 +168,41 @@ def test_stream_chunk_with_multiple_tool_calls_missing_fields(self): "role": "ASSISTANT", "content": None, "toolCalls": [ - { - "type": "FUNCTION", - "id": "call_1", - "name": "get_weather", - # Missing arguments - }, + {"type": "FUNCTION", "id": "call_1", "name": "get_weather"}, { "type": "FUNCTION", "name": "get_time", "arguments": '{"timezone": "UTC"}', - # Missing id }, { "type": "FUNCTION", "id": "call_3", "name": "calculate", "arguments": '{"expression": "2+2"}', - # Complete }, ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None assert len(result.choices[0].delta.tool_calls) == 3 - # First tool call - missing arguments assert result.choices[0].delta.tool_calls[0]["id"] == "call_1" assert ( result.choices[0].delta.tool_calls[0]["function"]["name"] == "get_weather" ) assert result.choices[0].delta.tool_calls[0]["function"]["arguments"] == "" - # Second tool call - missing id - assert result.choices[0].delta.tool_calls[1]["id"] == "" + assert result.choices[0].delta.tool_calls[1]["id"].startswith("call_") assert result.choices[0].delta.tool_calls[1]["function"]["name"] == "get_time" assert ( result.choices[0].delta.tool_calls[1]["function"]["arguments"] == '{"timezone": "UTC"}' ) - # Third tool call - complete assert result.choices[0].delta.tool_calls[2]["id"] == "call_3" assert result.choices[0].delta.tool_calls[2]["function"]["name"] == "calculate" assert ( @@ -279,10 +210,41 @@ def test_stream_chunk_with_multiple_tool_calls_missing_fields(self): == '{"expression": "2+2"}' ) - def test_stream_chunk_without_tool_calls(self): + def test_stream_chunk_missing_id_is_deterministic_across_chunks(self): """ - Test that streaming chunks without tool calls continue to work as before. + Two chunks emitting the same logical tool call (same name + arguments + at the same position) must receive the *same* synthesized id so the + downstream stream-merger does not treat them as distinct calls. + Random uuid4 per chunk would regress this — see bug ffdef760. """ + same_chunk_payload = lambda: { + "index": 0, + "finishReason": None, + "message": { + "role": "ASSISTANT", + "content": None, + "toolCalls": [ + { + "type": "FUNCTION", + "name": "get_weather", + "arguments": '{"location": "San Francisco"}', + } + ], + }, + } + + first = handle_generic_stream_chunk(same_chunk_payload()) + second = handle_generic_stream_chunk(same_chunk_payload()) + + assert first.choices[0].delta.tool_calls is not None + assert second.choices[0].delta.tool_calls is not None + first_id = first.choices[0].delta.tool_calls[0]["id"] + second_id = second.choices[0].delta.tool_calls[0]["id"] + assert first_id == second_id + assert first_id.startswith("call_") + + def test_stream_chunk_without_tool_calls(self): + """Plain text chunks (no tool calls) pass through correctly.""" chunk_data = { "index": 0, "finishReason": None, @@ -292,14 +254,7 @@ def test_stream_chunk_without_tool_calls(self): }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.content == "Hello, how can I help you?" diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py b/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py new file mode 100644 index 000000000000..30f49bea344e --- /dev/null +++ b/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py @@ -0,0 +1,406 @@ +""" +Unit tests for OCI Generative AI embedding transformation. + +These tests exercise the transformation layer only — no real OCI calls are made. +""" + +import json +import os +import sys +from typing import Any +from unittest.mock import MagicMock + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.embed.transformation import OCI_EMBED_BATCH_LIMIT, OCIEmbedConfig +from litellm.types.utils import EmbeddingResponse, Usage + +# --------------------------------------------------------------------------- +# Test fixtures +# --------------------------------------------------------------------------- + +COMPARTMENT_ID = "ocid1.compartment.oc1..test" +BASE_PARAMS = { + "oci_region": "us-ashburn-1", + "oci_user": "ocid1.user.oc1..test", + "oci_fingerprint": "aa:bb:cc:dd", + "oci_tenancy": "ocid1.tenancy.oc1..test", + "oci_compartment_id": COMPARTMENT_ID, + "oci_key": "-----BEGIN RSA PRIVATE KEY-----\nfakekey\n-----END RSA PRIVATE KEY-----", +} + + +class TestOCIEmbedConfig: + def _config(self) -> OCIEmbedConfig: + return OCIEmbedConfig() + + # ------------------------------------------------------------------ + # validate_environment + # ------------------------------------------------------------------ + + def test_validate_environment_sets_headers(self): + cfg = self._config() + headers = cfg.validate_environment( + headers={}, + model="oci/cohere.embed-v3.0", + messages=[], + optional_params=BASE_PARAMS, + litellm_params={}, + ) + assert headers["content-type"] == "application/json" + assert "litellm/" in headers["user-agent"] + + # ------------------------------------------------------------------ + # get_complete_url + # ------------------------------------------------------------------ + + def test_get_complete_url_default_region(self): + cfg = self._config() + url = cfg.get_complete_url( + api_base=None, + api_key=None, + model="cohere.embed-v3.0", + optional_params={"oci_region": "us-chicago-1"}, + litellm_params={}, + ) + assert ( + url + == "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText" + ) + + def test_get_complete_url_respects_api_base(self): + """api_base is treated as a base URL — the action path is appended.""" + cfg = self._config() + url = cfg.get_complete_url( + api_base="https://custom.endpoint.example.com", + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.endpoint.example.com/20231130/actions/embedText" + + def test_get_complete_url_strips_trailing_slash(self): + """Trailing slash is stripped from api_base before appending the action path.""" + cfg = self._config() + url = cfg.get_complete_url( + api_base="https://custom.endpoint.example.com/", + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.endpoint.example.com/20231130/actions/embedText" + + def test_get_complete_url_full_url_is_not_doubled(self): + """A fully-formed embedText URL must not have the action path appended twice.""" + cfg = self._config() + full_url = ( + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + "/20231130/actions/embedText" + ) + url = cfg.get_complete_url( + api_base=full_url, + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, + litellm_params={}, + ) + assert url == full_url + + # ------------------------------------------------------------------ + # transform_embedding_request + # ------------------------------------------------------------------ + + def test_transform_request_single_string(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input="hello world", + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert result["compartmentId"] == COMPARTMENT_ID + assert result["servingMode"]["servingType"] == "ON_DEMAND" + assert result["servingMode"]["modelId"] == "cohere.embed-v3.0" + assert result["inputs"] == ["hello world"] + + def test_transform_request_list_of_texts(self): + cfg = self._config() + texts = ["hello", "world"] + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=texts, + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert result["inputs"] == texts + + def test_transform_request_with_input_type(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["query"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "input_type": "SEARCH_QUERY", + }, + headers={}, + ) + assert result["inputType"] == "SEARCH_QUERY" + + def test_transform_request_with_output_dimensions(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v4.0", + input=["text"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "outputDimensions": 512, + }, + headers={}, + ) + assert result["outputDimensions"] == 512 + + def test_transform_request_dedicated_serving_mode(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["text"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "oci_serving_mode": "DEDICATED", + "oci_endpoint_id": "ocid1.genaiendpoint.oc1..test", + }, + headers={}, + ) + assert result["servingMode"]["servingType"] == "DEDICATED" + assert result["servingMode"]["endpointId"] == "ocid1.genaiendpoint.oc1..test" + assert "modelId" not in result["servingMode"] + + def test_transform_request_missing_compartment_id_raises(self): + cfg = self._config() + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["text"], + optional_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert "oci_compartment_id" in str(exc_info.value) + + def test_transform_request_batch_limit_exceeded_raises(self): + cfg = self._config() + texts = ["text"] * (OCI_EMBED_BATCH_LIMIT + 1) + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=texts, + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert str(OCI_EMBED_BATCH_LIMIT) in str(exc_info.value) + + def test_transform_request_invalid_serving_mode_raises(self): + cfg = self._config() + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["text"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "oci_serving_mode": "INVALID", + }, + headers={}, + ) + assert exc_info.value.status_code == 400 + + def test_transform_request_none_input_becomes_string(self): + """Non-list, non-string inputs are coerced to str.""" + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=42, # type: ignore + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert result["inputs"] == ["42"] + + # ------------------------------------------------------------------ + # transform_embedding_response + # ------------------------------------------------------------------ + + def _mock_response(self, status_code: int, body: dict) -> httpx.Response: + return httpx.Response( + status_code=status_code, + content=json.dumps(body).encode(), + headers={"content-type": "application/json"}, + ) + + def test_transform_response_success(self): + cfg = self._config() + model_response = EmbeddingResponse() + raw = self._mock_response( + 200, + { + "embeddings": [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], + "modelId": "cohere.embed-v3.0", + "modelVersion": "3.0.0", + # Actual OCI API returns per-input token counts + "inputTextTokenCounts": [5, 5], + }, + ) + result = cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["index"] == 1 + assert result.model == "cohere.embed-v3.0" + assert result.usage.prompt_tokens == 10 + + def test_transform_response_no_usage(self): + cfg = self._config() + model_response = EmbeddingResponse() + raw = self._mock_response( + 200, + { + "embeddings": [[0.1]], + "modelId": "cohere.embed-v3.0", + "modelVersion": "3.0.0", + }, + ) + result = cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + assert len(result.data) == 1 + + def test_transform_response_http_error_raises(self): + cfg = self._config() + raw = self._mock_response(401, {"error": "Unauthorized"}) + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=EmbeddingResponse(), + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + assert exc_info.value.status_code == 401 + + def test_transform_response_invalid_json_raises(self): + cfg = self._config() + raw = httpx.Response( + status_code=200, + content=b"not-json", + headers={"content-type": "text/plain"}, + ) + with pytest.raises(OCIError): + cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=EmbeddingResponse(), + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + + # ------------------------------------------------------------------ + # map_openai_params + # ------------------------------------------------------------------ + + def test_map_openai_params_dimensions(self): + cfg = self._config() + result = cfg.map_openai_params( + non_default_params={"dimensions": 512}, + optional_params={}, + model="cohere.embed-v4.0", + ) + assert result["outputDimensions"] == 512 + + def test_map_openai_params_encoding_format_not_supported(self): + """encoding_format is not a supported OCI param — it is silently ignored by map_openai_params. + + The litellm framework handles unsupported-param rejection above this layer, + based on get_supported_openai_params() not including 'encoding_format'. + """ + cfg = self._config() + result = cfg.map_openai_params( + non_default_params={"encoding_format": "float"}, + optional_params={}, + model="cohere.embed-v3.0", + ) + assert "encoding_format" not in result + + def test_map_openai_params_encoding_format_dropped_silently(self): + cfg = self._config() + result = cfg.map_openai_params( + non_default_params={"encoding_format": "float"}, + optional_params={}, + model="cohere.embed-v3.0", + drop_params=True, + ) + assert "encoding_format" not in result + + # ------------------------------------------------------------------ + # env var credential resolution + # ------------------------------------------------------------------ + + def test_env_var_compartment_id(self, monkeypatch): + monkeypatch.setenv("OCI_COMPARTMENT_ID", "ocid1.compartment.from.env") + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["hello"], + optional_params={}, # no compartment_id in params + headers={}, + ) + assert result["compartmentId"] == "ocid1.compartment.from.env" + + def test_explicit_param_overrides_env_var(self, monkeypatch): + monkeypatch.setenv("OCI_COMPARTMENT_ID", "ocid1.compartment.from.env") + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["hello"], + optional_params={"oci_compartment_id": "ocid1.compartment.explicit"}, + headers={}, + ) + assert result["compartmentId"] == "ocid1.compartment.explicit" + + def test_env_var_region_used_in_url(self, monkeypatch): + monkeypatch.setenv("OCI_REGION", "eu-frankfurt-1") + cfg = self._config() + url = cfg.get_complete_url( + api_base=None, + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, # no explicit region + litellm_params={}, + ) + assert "eu-frankfurt-1" in url diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py index 4ecca377e634..61c13ad62a19 100644 --- a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py +++ b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py @@ -76,7 +76,7 @@ def test_get_complete_url_custom_region(self): assert "embedText" in url def test_get_complete_url_custom_api_base(self): - """test_get_complete_url returns api_base as-is when provided.""" + """test_get_complete_url treats api_base as a base URL and appends the embedText path.""" config = OCIEmbeddingConfig() custom_base = "https://custom.oci.example.com/embed" url = config.get_complete_url( @@ -86,7 +86,7 @@ def test_get_complete_url_custom_api_base(self): optional_params={}, litellm_params={}, ) - assert url == custom_base + assert url == f"{custom_base}/20231130/actions/embedText" def test_get_supported_openai_params(self): """test_get_supported_openai_params returns expected params list.""" @@ -96,7 +96,7 @@ def test_get_supported_openai_params(self): assert "encoding_format" not in params def test_map_openai_params_dimensions(self): - """test dimensions is mapped correctly.""" + """test dimensions is mapped to outputDimensions (OCI API field name).""" config = OCIEmbeddingConfig() optional_params = {} result = config.map_openai_params( @@ -105,7 +105,8 @@ def test_map_openai_params_dimensions(self): model=TEST_MODEL_NAME, drop_params=False, ) - assert result["dimensions"] == 512 + assert result["outputDimensions"] == 512 + assert "dimensions" not in result def test_validate_environment_with_credentials(self, supplied_params): """test validate_environment returns content-type and user-agent headers when credentials are supplied.""" @@ -122,13 +123,15 @@ def test_validate_environment_with_credentials(self, supplied_params): assert "litellm" in result["user-agent"] def test_validate_environment_missing_credentials(self): - """test validate_environment raises Exception with 'Missing required parameters' when credentials are incomplete.""" + """test validate_environment raises OCIError when required credentials are missing.""" + from litellm.llms.oci.common_utils import OCIError + config = OCIEmbeddingConfig() incomplete_params = { "oci_user": "ocid1.user.oc1..xxx", # Missing oci_fingerprint, oci_tenancy, oci_key/oci_key_file, oci_compartment_id } - with pytest.raises(Exception) as excinfo: + with pytest.raises(OCIError, match="Missing required parameters"): config.validate_environment( headers={}, model=TEST_MODEL, @@ -136,7 +139,6 @@ def test_validate_environment_missing_credentials(self): optional_params=incomplete_params, litellm_params={}, ) - assert "Missing required parameters" in str(excinfo.value) def test_validate_environment_with_signer(self): """test validate_environment passes when oci_signer is provided.""" @@ -234,13 +236,15 @@ def test_transform_embedding_request_string_input(self): assert result["inputs"] == ["Hello world"] def test_transform_embedding_request_token_list_raises(self): - """test token-array inputs raise ValueError instead of silent conversion.""" + """test token-array inputs raise OCIError instead of silent conversion.""" + from litellm.llms.oci.common_utils import OCIError + config = OCIEmbeddingConfig() optional_params = { "oci_compartment_id": TEST_COMPARTMENT_ID, } with patch.object(config, "sign_request", return_value=({}, "{}")): - with pytest.raises(ValueError, match="does not support token-array"): + with pytest.raises(OCIError, match="does not support token-array"): config.transform_embedding_request( model=TEST_MODEL_NAME, input=[[1234, 5678]], @@ -264,6 +268,10 @@ def test_transform_embedding_response(self): raw_response=mock_response, model_response=model_response, logging_obj=mock_logging, + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, ) assert isinstance(result, EmbeddingResponse) @@ -296,6 +304,10 @@ def test_transform_embedding_response_error(self): raw_response=mock_response, model_response=model_response, logging_obj=mock_logging, + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, ) def test_model_prices_embedding_models(self): diff --git a/tests/test_litellm/llms/oci/rerank/__init__.py b/tests/test_litellm/llms/oci/rerank/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/test_litellm/llms/oci/test_oci_common_utils.py b/tests/test_litellm/llms/oci/test_oci_common_utils.py new file mode 100644 index 000000000000..d306d7351dda --- /dev/null +++ b/tests/test_litellm/llms/oci/test_oci_common_utils.py @@ -0,0 +1,521 @@ +""" +Unit tests for litellm/llms/oci/common_utils.py. + +Covers schema utilities, signing helpers, and credential resolution paths +that require no real OCI credentials or network calls. +""" + +import pytest +from unittest.mock import MagicMock, patch + +from litellm.llms.oci.common_utils import ( + OCI_API_VERSION, + OCIError, + OCIRequestWrapper, + build_signature_string, + enrich_cohere_param_description, + get_oci_base_url, + resolve_oci_credentials, + resolve_oci_schema_anyof, + resolve_oci_schema_refs, + sanitize_oci_schema, + sha256_base64, + sign_oci_request, + sign_with_oci_signer, + validate_oci_environment, +) + +# --------------------------------------------------------------------------- +# OCI_API_VERSION +# --------------------------------------------------------------------------- + + +def test_oci_api_version_constant(): + assert OCI_API_VERSION == "20231130" + + +# --------------------------------------------------------------------------- +# sha256_base64 +# --------------------------------------------------------------------------- + + +def test_sha256_base64_known_value(): + import base64, hashlib + + data = b"hello" + expected = base64.b64encode(hashlib.sha256(data).digest()).decode() + assert sha256_base64(data) == expected + + +def test_sha256_base64_empty(): + result = sha256_base64(b"") + assert isinstance(result, str) + assert len(result) > 0 + + +# --------------------------------------------------------------------------- +# build_signature_string +# --------------------------------------------------------------------------- + + +def test_build_signature_string_request_target(): + headers = {"host": "example.com", "date": "Mon, 01 Jan 2024 00:00:00 GMT"} + result = build_signature_string( + "POST", "/20231130/actions/chat", headers, ["(request-target)", "host", "date"] + ) + lines = result.split("\n") + assert lines[0] == "(request-target): post /20231130/actions/chat" + assert lines[1] == "host: example.com" + assert lines[2] == "date: Mon, 01 Jan 2024 00:00:00 GMT" + + +def test_build_signature_string_method_lowercased(): + headers = {"host": "h"} + result = build_signature_string("GET", "/path", headers, ["(request-target)"]) + assert result == "(request-target): get /path" + + +# --------------------------------------------------------------------------- +# OCIRequestWrapper.path_url +# --------------------------------------------------------------------------- + + +def test_request_wrapper_path_url_no_query(): + w = OCIRequestWrapper( + method="POST", + url="https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/20231130/actions/chat", + headers={}, + body=b"", + ) + assert w.path_url == "/20231130/actions/chat" + + +def test_request_wrapper_path_url_with_query(): + w = OCIRequestWrapper( + method="GET", + url="https://example.com/path?foo=bar&baz=1", + headers={}, + body=b"", + ) + assert w.path_url == "/path?foo=bar&baz=1" + + +# --------------------------------------------------------------------------- +# resolve_oci_credentials +# --------------------------------------------------------------------------- + + +def test_resolve_credentials_from_params(): + params = { + "oci_region": "eu-frankfurt-1", + "oci_user": "user1", + "oci_fingerprint": "fp1", + "oci_tenancy": "tenant1", + "oci_key": "key_content", + "oci_compartment_id": "comp1", + } + result = resolve_oci_credentials(params) + assert result["oci_region"] == "eu-frankfurt-1" + assert result["oci_user"] == "user1" + assert result["oci_compartment_id"] == "comp1" + + +def test_resolve_credentials_env_fallback(monkeypatch): + monkeypatch.setenv("OCI_REGION", "ap-tokyo-1") + monkeypatch.setenv("OCI_USER", "env_user") + monkeypatch.setenv("OCI_COMPARTMENT_ID", "env_comp") + result = resolve_oci_credentials({}) + assert result["oci_region"] == "ap-tokyo-1" + assert result["oci_user"] == "env_user" + assert result["oci_compartment_id"] == "env_comp" + + +def test_resolve_credentials_region_default(monkeypatch): + monkeypatch.delenv("OCI_REGION", raising=False) + result = resolve_oci_credentials({}) + assert result["oci_region"] == "us-ashburn-1" + + +def test_resolve_credentials_params_override_env(monkeypatch): + monkeypatch.setenv("OCI_REGION", "ap-tokyo-1") + result = resolve_oci_credentials({"oci_region": "us-phoenix-1"}) + assert result["oci_region"] == "us-phoenix-1" + + +# --------------------------------------------------------------------------- +# get_oci_base_url +# --------------------------------------------------------------------------- + + +def test_get_oci_base_url_explicit_api_base(): + url = get_oci_base_url({}, api_base="https://custom.endpoint.com/") + assert url == "https://custom.endpoint.com" + + +@pytest.mark.parametrize( + "api_base", + [ + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/chat", + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/chat/", + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText", + ], +) +def test_get_oci_base_url_strips_trailing_action_path(api_base): + assert ( + get_oci_base_url({}, api_base=api_base) + == "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + ) + + +def test_get_oci_base_url_from_region(): + url = get_oci_base_url({"oci_region": "eu-frankfurt-1"}) + assert url == "https://inference.generativeai.eu-frankfurt-1.oci.oraclecloud.com" + + +@pytest.mark.parametrize( + "region", + [ + "evil.com/#", + "evil.com", + "us-ashburn-1/../attacker", + "ATTACKER", + "-leading-hyphen", + "trailing-hyphen-", + "a", + "a" * 33, + "us ashburn 1", + "us_ashburn_1", + ], +) +def test_get_oci_base_url_rejects_unsafe_region(region): + with pytest.raises(OCIError, match="Invalid OCI region"): + get_oci_base_url({"oci_region": region}) + + +def test_get_oci_base_url_empty_region_falls_back_to_default(monkeypatch): + monkeypatch.delenv("OCI_REGION", raising=False) + url = get_oci_base_url({"oci_region": ""}) + assert url == "https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com" + + +@pytest.mark.parametrize( + "region", + [ + "us-ashburn-1", + "eu-frankfurt-1", + "ap-tokyo-1", + "us-chicago-1", + "us-phoenix-1", + "ap", + ], +) +def test_get_oci_base_url_accepts_valid_region(region): + url = get_oci_base_url({"oci_region": region}) + assert url == f"https://inference.generativeai.{region}.oci.oraclecloud.com" + + +# --------------------------------------------------------------------------- +# validate_oci_environment +# --------------------------------------------------------------------------- + + +def test_validate_oci_environment_sets_defaults(): + headers = {} + result = validate_oci_environment(headers, {}) + assert result["content-type"] == "application/json" + assert "user-agent" in result + + +def test_validate_oci_environment_does_not_overwrite_existing(): + headers = {"content-type": "text/plain", "user-agent": "my-agent"} + result = validate_oci_environment(headers, {}) + assert result["content-type"] == "text/plain" + assert result["user-agent"] == "my-agent" + + +# --------------------------------------------------------------------------- +# sign_with_oci_signer — error paths +# --------------------------------------------------------------------------- + + +def test_sign_with_oci_signer_none_raises(): + with pytest.raises(ValueError, match="oci_signer cannot be None"): + sign_with_oci_signer({}, {"oci_signer": None}, {}, "https://example.com") + + +def test_sign_with_oci_signer_exception_wrapped(): + bad_signer = MagicMock() + bad_signer.do_request_sign.side_effect = RuntimeError("signing failed") + with pytest.raises(OCIError, match="Failed to sign request"): + sign_with_oci_signer( + {}, {"oci_signer": bad_signer}, {"key": "val"}, "https://example.com" + ) + + +def test_sign_with_oci_signer_success(): + signer = MagicMock() + signer.do_request_sign.return_value = None + headers, body = sign_with_oci_signer( + {}, {"oci_signer": signer}, {"key": "val"}, "https://example.com" + ) + assert isinstance(body, bytes) + signer.do_request_sign.assert_called_once() + + +# --------------------------------------------------------------------------- +# sign_oci_request — routing +# --------------------------------------------------------------------------- + + +def test_sign_oci_request_routes_to_signer(): + signer = MagicMock() + signer.do_request_sign.return_value = None + headers, body = sign_oci_request( + {}, {"oci_signer": signer}, {}, "https://example.com" + ) + signer.do_request_sign.assert_called_once() + + +def test_sign_oci_request_routes_to_manual_missing_creds(): + with pytest.raises(OCIError, match="Missing required OCI credentials"): + sign_oci_request({}, {}, {}, "https://example.com") + + +# --------------------------------------------------------------------------- +# load_private_key_from_file — error paths (no real key needed) +# --------------------------------------------------------------------------- + + +def test_load_private_key_from_file_not_found(): + from litellm.llms.oci.common_utils import load_private_key_from_file + + with pytest.raises(FileNotFoundError, match="Private key file not found"): + load_private_key_from_file("/nonexistent/path/key.pem") + + +def test_load_private_key_from_file_empty(tmp_path): + from litellm.llms.oci.common_utils import load_private_key_from_file + + empty = tmp_path / "empty.pem" + empty.write_text("") + with pytest.raises(ValueError, match="Private key file is empty"): + load_private_key_from_file(str(empty)) + + +def test_load_private_key_from_file_os_error(): + from litellm.llms.oci.common_utils import load_private_key_from_file + + with patch("builtins.open", side_effect=OSError("permission denied")): + with pytest.raises(OSError, match="Failed to read private key file"): + load_private_key_from_file("/some/path/key.pem") + + +# --------------------------------------------------------------------------- +# resolve_oci_schema_refs +# --------------------------------------------------------------------------- + + +def test_resolve_schema_refs_basic(): + schema = { + "$defs": {"Foo": {"type": "string"}}, + "properties": {"x": {"$ref": "#/$defs/Foo"}}, + } + result = resolve_oci_schema_refs(schema) + assert result["properties"]["x"] == {"type": "string"} + assert "$defs" not in result + + +def test_resolve_schema_refs_external_ref_unchanged(): + schema = {"properties": {"x": {"$ref": "https://example.com/schema"}}} + result = resolve_oci_schema_refs(schema) + assert result["properties"]["x"] == {"$ref": "https://example.com/schema"} + + +def test_resolve_schema_refs_circular_breaks_cycle(): + schema = { + "$defs": {"Node": {"properties": {"child": {"$ref": "#/$defs/Node"}}}}, + "properties": {"root": {"$ref": "#/$defs/Node"}}, + } + result = resolve_oci_schema_refs(schema) + # Should not raise; circular ref replaced with {"type": "object"} + child = result["properties"]["root"]["properties"]["child"] + assert child == {"type": "object"} + + +def test_resolve_schema_refs_no_defs(): + schema = {"type": "object", "properties": {"x": {"type": "string"}}} + result = resolve_oci_schema_refs(schema) + assert result == schema + + +# --------------------------------------------------------------------------- +# resolve_oci_schema_anyof +# --------------------------------------------------------------------------- + + +def test_resolve_schema_anyof_optional_field(): + schema = {"anyOf": [{"type": "string"}, {"type": "null"}]} + result = resolve_oci_schema_anyof(schema) + assert result["type"] == "string" + assert "anyOf" not in result + + +def test_resolve_schema_anyof_all_null_returns_empty(): + schema = {"anyOf": [{"type": "null"}, {"type": "null"}]} + result = resolve_oci_schema_anyof(schema) + # No non-null branch — anyOf stays or schema unchanged + # The function only strips anyOf when there IS a non-null branch + assert "anyOf" in result + + +def test_resolve_schema_anyof_no_anyof_unchanged(): + schema = {"type": "string", "description": "A name"} + assert resolve_oci_schema_anyof(schema) == schema + + +def test_resolve_schema_anyof_nested(): + schema = {"properties": {"age": {"anyOf": [{"type": "integer"}, {"type": "null"}]}}} + result = resolve_oci_schema_anyof(schema) + assert result["properties"]["age"]["type"] == "integer" + + +# --------------------------------------------------------------------------- +# sanitize_oci_schema +# --------------------------------------------------------------------------- + + +def test_sanitize_schema_removes_title(): + schema = {"title": "MyModel", "type": "object", "properties": {}} + result = sanitize_oci_schema(schema) + assert "title" not in result + + +def test_sanitize_schema_removes_null_default(): + schema = {"type": "string", "default": None} + result = sanitize_oci_schema(schema) + assert "default" not in result + + +def test_sanitize_schema_keeps_non_null_default(): + schema = {"type": "string", "default": "hello"} + result = sanitize_oci_schema(schema) + assert result["default"] == "hello" + + +def test_sanitize_schema_type_any_becomes_object(): + schema = {"type": "any"} + result = sanitize_oci_schema(schema) + assert result["type"] == "object" + + +def test_sanitize_schema_type_list_picks_non_null(): + schema = {"type": ["string", "null"]} + result = sanitize_oci_schema(schema) + assert result["type"] == "string" + + +def test_sanitize_schema_type_list_all_null_becomes_string(): + schema = {"type": ["null"]} + result = sanitize_oci_schema(schema) + assert result["type"] == "string" + + +def test_sanitize_schema_array_gets_items(): + schema = {"type": "array"} + result = sanitize_oci_schema(schema) + assert result["items"] == {"type": "object"} + + +def test_sanitize_schema_array_keeps_existing_items(): + schema = {"type": "array", "items": {"type": "string"}} + result = sanitize_oci_schema(schema) + assert result["items"] == {"type": "string"} + + +def test_sanitize_schema_required_filters_missing_properties(): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a", "b"], # "b" not in properties + } + result = sanitize_oci_schema(schema) + assert result["required"] == ["a"] + + +def test_sanitize_schema_required_non_list_becomes_empty(): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": "a", # invalid: string instead of list + } + result = sanitize_oci_schema(schema) + assert result["required"] == [] + + +def test_sanitize_schema_list_input(): + schemas = [{"title": "A", "type": "string"}, {"title": "B", "type": "integer"}] + result = sanitize_oci_schema(schemas) + assert all("title" not in s for s in result) + + +# --------------------------------------------------------------------------- +# enrich_cohere_param_description +# --------------------------------------------------------------------------- + + +def test_enrich_description_enum(): + result = enrich_cohere_param_description("A color", {"enum": ["red", "blue"]}) + assert "Allowed values: ['red', 'blue']" in result + + +def test_enrich_description_format(): + result = enrich_cohere_param_description("A date", {"format": "date-time"}) + assert "Format: date-time" in result + + +def test_enrich_description_range_both(): + result = enrich_cohere_param_description("A number", {"minimum": 0, "maximum": 100}) + assert "Range: min=0, max=100" in result + + +def test_enrich_description_range_min_only(): + result = enrich_cohere_param_description("A number", {"minimum": 1}) + assert "Range: min=1" in result + assert "max" not in result + + +def test_enrich_description_range_max_only(): + result = enrich_cohere_param_description("", {"maximum": 10}) + assert "Range: max=10" in result + + +def test_enrich_description_pattern(): + result = enrich_cohere_param_description("An ID", {"pattern": "^[a-z]+$"}) + assert "Pattern: ^[a-z]+$" in result + + +def test_enrich_description_all_constraints(): + result = enrich_cohere_param_description( + "Val", + { + "enum": ["a"], + "format": "uuid", + "minimum": 0, + "maximum": 1, + "pattern": ".*", + }, + ) + assert "Allowed values" in result + assert "Format" in result + assert "Range" in result + assert "Pattern" in result + + +def test_enrich_description_no_constraints(): + result = enrich_cohere_param_description("Just a description", {}) + assert result == "Just a description" + + +def test_enrich_description_empty_description_no_constraints(): + result = enrich_cohere_param_description("", {}) + assert result == "" diff --git a/tests/test_litellm/llms/oci/test_oci_coverage_boost.py b/tests/test_litellm/llms/oci/test_oci_coverage_boost.py new file mode 100644 index 000000000000..0b7afa3775dd --- /dev/null +++ b/tests/test_litellm/llms/oci/test_oci_coverage_boost.py @@ -0,0 +1,1152 @@ +""" +Coverage-boost tests for the OCI provider happy paths. + +Covers: + - litellm/llms/oci/common_utils.py (sign_with_manual_credentials, routing) + - litellm/llms/oci/chat/generic.py (message adaptation, tool conversion, streaming) + - litellm/llms/oci/chat/cohere.py (message adaptation, response parsing, streaming) + - litellm/llms/oci/chat/transformation.py (OCIChatConfig methods, stream wrappers) + +All tests are self-contained and require no real OCI credentials or network access. +""" + +import json +import pytest +from unittest.mock import patch, MagicMock, AsyncMock + +import httpx + +from litellm import ModelResponse +from litellm.llms.oci.chat.cohere import ( + _extract_text_content, + adapt_messages_to_cohere_standard, + handle_cohere_response, + handle_cohere_stream_chunk, +) +from litellm.llms.oci.chat.generic import ( + adapt_messages_to_generic_oci_standard, + adapt_messages_to_generic_oci_standard_tool_response, + adapt_tool_definition_to_oci_standard, + adapt_tools_to_openai_standard, + handle_generic_stream_chunk, +) +from litellm.llms.oci.chat.transformation import OCIChatConfig, get_vendor_from_model +from litellm.llms.oci.common_utils import ( + OCIError, + sign_with_manual_credentials, + sign_oci_request, + validate_oci_environment, +) +from litellm.types.llms.oci import OCIVendors, OCIToolCall + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +_MANUAL_CREDS = { + "oci_user": "ocid1.user.oc1..xxx", + "oci_fingerprint": "aa:bb:cc:dd", + "oci_tenancy": "ocid1.tenancy.oc1..xxx", + "oci_compartment_id": "ocid1.compartment.oc1..xxx", + "oci_key": "-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----", +} + +_API_BASE = "https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/20231130/actions/chat" + +_COHERE_MODEL = "cohere.command-r-plus" +_GENERIC_MODEL = "meta.llama-3-70b-instruct" + + +# =========================================================================== +# common_utils.py — sign_with_manual_credentials happy paths +# =========================================================================== + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_str") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_with_manual_credentials_inline_key( + mock_hashes, mock_padding, mock_load_key +): + """sign_with_manual_credentials succeeds with an inline oci_key string.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"fake_signature" + mock_load_key.return_value = mock_key + + result_headers, body = sign_with_manual_credentials( + {}, _MANUAL_CREDS, {"key": "val"}, _API_BASE + ) + + assert "authorization" in result_headers + assert result_headers["authorization"].startswith('Signature version="1"') + assert "rsa-sha256" in result_headers["authorization"] + assert isinstance(body, bytes) + mock_key.sign.assert_called_once() + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_file") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_with_manual_credentials_key_file( + mock_hashes, mock_padding, mock_load_file +): + """sign_with_manual_credentials falls back to oci_key_file when oci_key absent.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"sig_from_file" + mock_load_file.return_value = mock_key + + creds = {**_MANUAL_CREDS, "oci_key_file": "/tmp/key.pem"} + creds_no_inline = {k: v for k, v in creds.items() if k != "oci_key"} + + result_headers, body = sign_with_manual_credentials( + {}, creds_no_inline, {}, _API_BASE + ) + + assert "authorization" in result_headers + mock_load_file.assert_called_once_with("/tmp/key.pem") + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_str") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_with_manual_credentials_authorization_contains_key_id( + mock_hashes, mock_padding, mock_load_key +): + """Authorization header encodes tenancy/user/fingerprint as key ID.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"sig" + mock_load_key.return_value = mock_key + + result_headers, _ = sign_with_manual_credentials({}, _MANUAL_CREDS, {}, _API_BASE) + + auth = result_headers["authorization"] + assert 'keyId="ocid1.tenancy.oc1..xxx/ocid1.user.oc1..xxx/aa:bb:cc:dd"' in auth + + +def test_sign_with_manual_credentials_non_string_oci_key_raises(): + """Passing a non-string oci_key raises OCIError(400).""" + bad_creds = {**_MANUAL_CREDS, "oci_key": 12345} + with pytest.raises(OCIError) as exc_info: + sign_with_manual_credentials({}, bad_creds, {}, _API_BASE) + assert exc_info.value.status_code == 400 + assert "oci_key must be a string" in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# common_utils.py — sign_oci_request routing +# --------------------------------------------------------------------------- + + +def test_sign_oci_request_routes_to_signer_when_present(): + """sign_oci_request delegates to sign_with_oci_signer when oci_signer is set.""" + signer = MagicMock() + signer.do_request_sign.return_value = None + headers, body = sign_oci_request({}, {"oci_signer": signer}, {"data": 1}, _API_BASE) + signer.do_request_sign.assert_called_once() + assert isinstance(body, bytes) + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_str") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_oci_request_routes_to_manual_when_no_signer( + mock_hashes, mock_padding, mock_load_key +): + """sign_oci_request delegates to sign_with_manual_credentials when oci_signer absent.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"sig" + mock_load_key.return_value = mock_key + + headers, body = sign_oci_request({}, _MANUAL_CREDS, {}, _API_BASE) + assert "authorization" in headers + + +# --------------------------------------------------------------------------- +# common_utils.py — _require_cryptography happy path +# --------------------------------------------------------------------------- + + +def test_require_cryptography_available_does_not_raise(): + """_require_cryptography() should not raise when the package is importable.""" + from litellm.llms.oci.common_utils import _require_cryptography + + with patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True): + _require_cryptography() # must not raise + + +# =========================================================================== +# generic.py — adapt_messages_to_generic_oci_standard +# =========================================================================== + + +def test_adapt_generic_user_message_string_content(): + messages = [{"role": "user", "content": "Hello!"}] + result = adapt_messages_to_generic_oci_standard(messages) + assert len(result) == 1 + assert result[0].role == "USER" + assert result[0].content[0].text == "Hello!" + + +def test_adapt_generic_assistant_message(): + messages = [{"role": "assistant", "content": "Hi there!"}] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "ASSISTANT" + assert result[0].content[0].text == "Hi there!" + + +def test_adapt_generic_system_message(): + messages = [{"role": "system", "content": "You are a helpful assistant."}] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "SYSTEM" + assert result[0].content[0].text == "You are a helpful assistant." + + +def test_adapt_generic_tool_message(): + messages = [ + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": "42 degrees", + } + ] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "TOOL" + assert result[0].toolCallId == "call_abc123" + assert result[0].content[0].text == "42 degrees" + + +def test_adapt_generic_assistant_tool_call_message(): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_xyz", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Rome"}', + }, + } + ], + } + ] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "ASSISTANT" + assert result[0].toolCalls is not None + assert len(result[0].toolCalls) == 1 + tc = result[0].toolCalls[0] + assert tc.name == "get_weather" + assert tc.arguments == '{"city": "Rome"}' + + +def test_adapt_generic_multipart_content(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Look at this:"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/img.png"}, + }, + ], + } + ] + result = adapt_messages_to_generic_oci_standard(messages) + assert len(result[0].content) == 2 + assert result[0].content[0].text == "Look at this:" + assert result[0].content[1].imageUrl.url == "https://example.com/img.png" + + +# --------------------------------------------------------------------------- +# generic.py — adapt_messages_to_generic_oci_standard_tool_response +# --------------------------------------------------------------------------- + + +def test_adapt_generic_tool_response_direct(): + result = adapt_messages_to_generic_oci_standard_tool_response( + "tool", "call_999", "The answer is 42" + ) + assert result.role == "TOOL" + assert result.toolCallId == "call_999" + assert result.content[0].text == "The answer is 42" + + +# --------------------------------------------------------------------------- +# generic.py — adapt_tool_definition_to_oci_standard +# --------------------------------------------------------------------------- + + +def test_adapt_tool_definition_basic(): + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Retrieve current weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + result = adapt_tool_definition_to_oci_standard(tools, OCIVendors.GENERIC) + assert len(result) == 1 + tool_def = result[0] + assert tool_def.name == "get_weather" + assert tool_def.type == "FUNCTION" + assert tool_def.parameters is not None + + +def test_adapt_tool_definition_resolves_refs(): + """$ref/$defs schemas are inlined before being sent to OCI.""" + tools = [ + { + "type": "function", + "function": { + "name": "do_thing", + "parameters": { + "$defs": {"Loc": {"type": "string"}}, + "type": "object", + "properties": {"location": {"$ref": "#/$defs/Loc"}}, + }, + }, + } + ] + result = adapt_tool_definition_to_oci_standard(tools, OCIVendors.GENERIC) + props = result[0].parameters["properties"] + assert props["location"] == {"type": "string"} + + +# --------------------------------------------------------------------------- +# generic.py — adapt_tools_to_openai_standard +# --------------------------------------------------------------------------- + + +def test_adapt_tools_to_openai_standard(): + oci_tool = OCIToolCall( + id="call_abc", + type="FUNCTION", + name="search", + arguments='{"query": "hello"}', + ) + result = adapt_tools_to_openai_standard([oci_tool]) + assert len(result) == 1 + assert result[0].id == "call_abc" + assert result[0].type == "function" + assert result[0].function["name"] == "search" + + +def test_adapt_tools_to_openai_standard_generates_id_when_absent(): + oci_tool = OCIToolCall( + id=None, + type="FUNCTION", + name="lookup", + arguments="{}", + ) + result = adapt_tools_to_openai_standard([oci_tool]) + assert result[0].id.startswith("call_") + + +# --------------------------------------------------------------------------- +# generic.py — handle_generic_stream_chunk +# --------------------------------------------------------------------------- + + +def test_handle_generic_stream_chunk_text_content(): + chunk = { + "message": { + "content": [{"type": "TEXT", "text": "Hello from OCI"}], + "role": "ASSISTANT", + }, + "finishReason": None, + "index": 0, + } + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].delta.content == "Hello from OCI" + assert result.choices[0].finish_reason is None + + +def test_handle_generic_stream_chunk_complete_finish_reason(): + chunk = {"finishReason": "COMPLETE", "index": 0} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "stop" + + +def test_handle_generic_stream_chunk_max_tokens_finish_reason(): + chunk = {"finishReason": "MAX_TOKENS", "index": 0} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "length" + + +def test_handle_generic_stream_chunk_tool_calls_finish_reason(): + chunk = {"finishReason": "TOOL_CALLS", "index": 0} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "tool_calls" + + +def test_handle_generic_stream_chunk_no_message(): + """Chunks without a message key should still parse without error.""" + chunk = {"finishReason": "COMPLETE", "index": 1} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].delta.content is None + assert result.choices[0].finish_reason == "stop" + + +# =========================================================================== +# cohere.py — _extract_text_content +# =========================================================================== + + +def test_extract_text_content_none(): + assert _extract_text_content(None) == "" + + +def test_extract_text_content_string(): + assert _extract_text_content("hello") == "hello" + + +def test_extract_text_content_list(): + content = [ + {"type": "text", "text": "foo"}, + {"type": "text", "text": "bar"}, + ] + assert _extract_text_content(content) == "foobar" + + +def test_extract_text_content_list_skips_non_text(): + content = [ + {"type": "image_url", "url": "https://x.com/img.png"}, + {"type": "text", "text": "only this"}, + ] + assert _extract_text_content(content) == "only this" + + +def test_extract_text_content_non_string_non_list(): + assert _extract_text_content(42) == "42" + + +# =========================================================================== +# cohere.py — adapt_messages_to_cohere_standard +# =========================================================================== + + +def test_adapt_cohere_user_in_history(): + messages = [ + {"role": "user", "content": "first question"}, + {"role": "user", "content": "current question"}, + ] + history = adapt_messages_to_cohere_standard(messages) + assert len(history) == 1 + assert history[0].role == "USER" + assert history[0].message == "first question" + + +def test_adapt_cohere_assistant_in_history(): + messages = [ + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "answer"}, + {"role": "user", "content": "follow-up"}, + ] + history = adapt_messages_to_cohere_standard(messages) + assert len(history) == 2 + chatbot_msg = history[1] + assert chatbot_msg.role == "CHATBOT" + assert chatbot_msg.message == "answer" + + +def test_adapt_cohere_assistant_with_tool_calls_in_history(): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "calc", "arguments": '{"x": 1}'}, + } + ], + }, + {"role": "user", "content": "thanks"}, + ] + history = adapt_messages_to_cohere_standard(messages) + assert len(history) == 1 + assert history[0].role == "CHATBOT" + assert history[0].toolCalls is not None + assert history[0].toolCalls[0].name == "calc" + + +def test_adapt_cohere_tool_result_in_history(): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "calc", "arguments": '{"x": 1}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "result: 42", + }, + {"role": "user", "content": "ok"}, + ] + history = adapt_messages_to_cohere_standard(messages) + tool_msg = next(m for m in history if m.role == "TOOL") + assert tool_msg.toolResults[0].call.name == "calc" + assert tool_msg.toolResults[0].outputs[0]["output"] == "result: 42" + + +# =========================================================================== +# cohere.py — handle_cohere_response +# =========================================================================== + + +_COHERE_RESPONSE_JSON = { + "modelId": "cohere.command-r-plus", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "COHERE", + "text": "Hello from Cohere!", + "finishReason": "COMPLETE", + "usage": { + "promptTokens": 10, + "completionTokens": 5, + "totalTokens": 15, + }, + }, +} + + +_COHERE_RAW_RESPONSE = httpx.Response(200, request=httpx.Request("POST", "https://oci")) + + +def test_handle_cohere_response_complete(): + model_response = ModelResponse() + result = handle_cohere_response( + _COHERE_RESPONSE_JSON, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.choices[0].finish_reason == "stop" + assert result.choices[0].message["content"] == "Hello from Cohere!" + assert result.usage.prompt_tokens == 10 + + +def test_handle_cohere_response_max_tokens(): + resp = { + **_COHERE_RESPONSE_JSON, + "chatResponse": { + **_COHERE_RESPONSE_JSON["chatResponse"], + "finishReason": "MAX_TOKENS", + }, + } + model_response = ModelResponse() + result = handle_cohere_response( + resp, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.choices[0].finish_reason == "length" + + +def test_handle_cohere_response_tool_call(): + resp = { + **_COHERE_RESPONSE_JSON, + "chatResponse": { + **_COHERE_RESPONSE_JSON["chatResponse"], + "finishReason": "TOOL_CALL", + "toolCalls": [{"name": "get_time", "parameters": {"tz": "UTC"}}], + }, + } + model_response = ModelResponse() + result = handle_cohere_response( + resp, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.choices[0].finish_reason == "tool_calls" + tool_calls = result.choices[0].message["tool_calls"] + assert tool_calls is not None + assert tool_calls[0]["function"]["name"] == "get_time" + + +def test_handle_cohere_response_missing_usage(): + resp = { + **_COHERE_RESPONSE_JSON, + "chatResponse": { + k: v + for k, v in _COHERE_RESPONSE_JSON["chatResponse"].items() + if k != "usage" + }, + } + model_response = ModelResponse() + result = handle_cohere_response( + resp, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.usage.prompt_tokens == 0 + assert result.usage.completion_tokens == 0 + assert result.usage.total_tokens == 0 + + +def test_handle_cohere_response_malformed_raises_oci_error(): + bad_json = {"chatResponse": {"apiFormat": "COHERE"}} + raw = httpx.Response(502, request=httpx.Request("POST", "https://oci")) + model_response = ModelResponse() + with pytest.raises(OCIError) as exc_info: + handle_cohere_response(bad_json, _COHERE_MODEL, model_response, raw) + assert exc_info.value.status_code == 502 + + +# =========================================================================== +# cohere.py — handle_cohere_stream_chunk +# =========================================================================== + + +def test_handle_cohere_stream_chunk_text(): + chunk = {"apiFormat": "COHERE", "text": "streaming text", "finishReason": None} + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].delta.content == "streaming text" + assert result.choices[0].finish_reason is None + + +def test_handle_cohere_stream_chunk_complete(): + # Real OCI Cohere terminal events carry the full response in `text` plus a + # populated `chatHistory`; the parser must drop that text to avoid doubling + # — but only when prior chunks already emitted the text as incremental + # deltas (signalled by ``prior_text_emitted=True``). + chunk = { + "apiFormat": "COHERE", + "text": "How can I help you today?", + "finishReason": "COMPLETE", + "chatHistory": [ + {"role": "USER", "message": "Hello!"}, + {"role": "CHATBOT", "message": "How can I help you today?"}, + ], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=True) + assert result.choices[0].finish_reason == "stop" + assert result.choices[0].delta.content is None + + +def test_handle_cohere_stream_chunk_max_tokens(): + chunk = { + "apiFormat": "COHERE", + "text": "truncated full response", + "finishReason": "MAX_TOKENS", + "chatHistory": [{"role": "CHATBOT", "message": "truncated full response"}], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=True) + assert result.choices[0].finish_reason == "length" + assert result.choices[0].delta.content is None + + +def test_handle_cohere_stream_chunk_tool_call(): + chunk = { + "apiFormat": "COHERE", + "text": "", + "finishReason": "TOOL_CALL", + "chatHistory": [{"role": "CHATBOT", "message": ""}], + } + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].finish_reason == "tool_calls" + assert not result.choices[0].delta.content + + +def test_handle_cohere_stream_chunk_terminal_drops_full_response_text(): + """Regression for double-output on cohere.command-* streaming. + + OCI's terminal SSE event re-sends the full assembled response in `text` + alongside a populated `chatHistory`. That text must be dropped — otherwise + it gets concatenated onto the already-streamed incremental deltas. The + caller signals "prior deltas already emitted text" via + ``prior_text_emitted=True``. + """ + chunk = { + "apiFormat": "COHERE", + "text": "How can I help you today?", + "finishReason": "COMPLETE", + "chatHistory": [ + {"role": "USER", "message": "Hello!"}, + {"role": "CHATBOT", "message": "How can I help you today?"}, + ], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=True) + assert result.choices[0].delta.content is None + + +def test_handle_cohere_stream_chunk_single_event_stream_preserves_text(): + """Degenerate single-event stream: the terminal chunk carries the only copy + of the response text. Without prior text deltas, suppressing here would + discard the response entirely — so the text must pass through.""" + chunk = { + "apiFormat": "COHERE", + "text": "Short answer.", + "finishReason": "COMPLETE", + "chatHistory": [{"role": "CHATBOT", "message": "Short answer."}], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=False) + assert result.choices[0].delta.content == "Short answer." + assert result.choices[0].finish_reason == "stop" + + +def test_handle_cohere_stream_chunk_incremental_passes_text_through(): + """Non-terminal chunks (no chatHistory) must emit their incremental text.""" + chunk = { + "apiFormat": "COHERE", + "text": "How can I ", + "finishReason": None, + } + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].delta.content == "How can I " + assert result.choices[0].finish_reason is None + + +def test_handle_cohere_stream_chunk_finish_reason_without_chathistory_keeps_text(): + """`finishReason` alone (no `chatHistory`) must NOT trigger the drop — + `chatHistory` is the discriminator for the consolidated terminal event.""" + chunk = { + "apiFormat": "COHERE", + "text": "tail delta", + "finishReason": "COMPLETE", + } + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].delta.content == "tail delta" + assert result.choices[0].finish_reason == "stop" + + +# =========================================================================== +# transformation.py — get_vendor_from_model +# =========================================================================== + + +def test_get_vendor_cohere(): + assert get_vendor_from_model("cohere.command-r-plus") == OCIVendors.COHERE + + +def test_get_vendor_generic_llama(): + assert get_vendor_from_model("meta.llama-3-70b-instruct") == OCIVendors.GENERIC + + +def test_get_vendor_generic_xai(): + assert get_vendor_from_model("xai.grok-4") == OCIVendors.GENERIC + + +def test_get_vendor_generic_google(): + assert get_vendor_from_model("google.gemini-2-flash") == OCIVendors.GENERIC + + +# =========================================================================== +# transformation.py — OCIChatConfig methods +# =========================================================================== + + +class TestOCIChatConfigGetCompleteUrl: + def test_returns_chat_endpoint_from_region(self): + config = OCIChatConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model=_GENERIC_MODEL, + optional_params={"oci_region": "eu-frankfurt-1"}, + litellm_params={}, + ) + assert url == ( + "https://inference.generativeai.eu-frankfurt-1.oci.oraclecloud.com" + "/20231130/actions/chat" + ) + + def test_respects_explicit_api_base(self): + config = OCIChatConfig() + url = config.get_complete_url( + api_base="https://custom.endpoint.com/", + api_key=None, + model=_GENERIC_MODEL, + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.endpoint.com/20231130/actions/chat" + + def test_full_chat_url_is_not_doubled(self): + config = OCIChatConfig() + full_url = ( + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + "/20231130/actions/chat" + ) + url = config.get_complete_url( + api_base=full_url, + api_key=None, + model=_GENERIC_MODEL, + optional_params={}, + litellm_params={}, + ) + assert url == full_url + + +class TestOCIChatConfigGetErrorClass: + def test_returns_oci_error(self): + config = OCIChatConfig() + err = config.get_error_class("boom", 503, {}) + assert isinstance(err, OCIError) + assert err.status_code == 503 + + +class TestOCIChatConfigSignRequest: + @patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) + @patch("litellm.llms.oci.common_utils.load_private_key_from_str") + @patch("litellm.llms.oci.common_utils.padding") + @patch("litellm.llms.oci.common_utils.hashes") + def test_sign_request_delegates(self, mock_hashes, mock_padding, mock_load_key): + mock_key = MagicMock() + mock_key.sign.return_value = b"sig" + mock_load_key.return_value = mock_key + + config = OCIChatConfig() + headers, body = config.sign_request( + headers={}, + optional_params=_MANUAL_CREDS, + request_data={"hello": "world"}, + api_base=_API_BASE, + ) + assert "authorization" in headers + assert isinstance(body, bytes) + + +class TestOCIChatConfigValidateEnvironment: + def test_with_signer_skips_credential_check(self): + """If oci_signer is provided, validate_environment must NOT raise.""" + config = OCIChatConfig() + signer = MagicMock() + result = config.validate_environment( + headers={}, + model=_GENERIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + optional_params={"oci_signer": signer}, + litellm_params={}, + ) + assert result["content-type"] == "application/json" + + def test_raises_when_messages_empty(self): + config = OCIChatConfig() + with pytest.raises(OCIError) as exc_info: + config.validate_environment( + headers={}, + model=_GENERIC_MODEL, + messages=[], + optional_params={"oci_signer": MagicMock()}, + litellm_params={}, + ) + assert exc_info.value.status_code == 400 + + +class TestOCIChatConfigGetOptionalParams: + def _config(self): + return OCIChatConfig() + + def test_cohere_maps_stop_to_stop_sequences(self): + config = self._config() + result = config._get_optional_params(OCIVendors.COHERE, {"stop": ["END"]}) + assert "stopSequences" in result + assert result["stopSequences"] == ["END"] + + def test_generic_maps_max_tokens(self): + config = self._config() + result = config._get_optional_params(OCIVendors.GENERIC, {"max_tokens": 512}) + assert result["maxTokens"] == 512 + + def test_tool_choice_string_auto_converted_to_dict(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": "auto"} + ) + assert result["toolChoice"] == {"type": "AUTO"} + + def test_tool_choice_string_none_converted_to_dict(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": "none"} + ) + assert result["toolChoice"] == {"type": "NONE"} + + def test_tool_choice_string_required_converted_to_dict(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": "required"} + ) + assert result["toolChoice"] == {"type": "REQUIRED"} + + def test_tool_choice_openai_function_dict_converted_to_oci_form(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, + { + "tool_choice": { + "type": "function", + "function": {"name": "my_func"}, + } + }, + ) + assert result["toolChoice"] == {"type": "FUNCTION", "name": "my_func"} + + def test_tool_choice_flat_function_dict_uppercased(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, + {"tool_choice": {"type": "function", "name": "my_func"}}, + ) + assert result["toolChoice"] == {"type": "FUNCTION", "name": "my_func"} + + def test_tool_choice_dict_auto_uppercased(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": {"type": "auto"}} + ) + assert result["toolChoice"] == {"type": "AUTO"} + + def test_response_format_json_generic(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"response_format": {"type": "json_object"}} + ) + assert result["responseFormat"]["type"] == "JSON_OBJECT" + + def test_tools_adapted_for_cohere(self): + config = self._config() + tools = [ + { + "type": "function", + "function": { + "name": "echo", + "description": "echo", + "parameters": { + "type": "object", + "properties": {"msg": {"type": "string"}}, + "required": ["msg"], + }, + }, + } + ] + result = config._get_optional_params(OCIVendors.COHERE, {"tools": tools}) + # tools should be CohereTool objects + assert len(result["tools"]) == 1 + assert result["tools"][0].name == "echo" + + def test_tools_adapted_for_generic(self): + config = self._config() + tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "search the web", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + }, + } + ] + result = config._get_optional_params(OCIVendors.GENERIC, {"tools": tools}) + assert len(result["tools"]) == 1 + assert result["tools"][0].name == "search" + + +class TestOCIChatConfigTransformRequest: + _base_params = {**_MANUAL_CREDS} + + def test_generic_model_transform(self): + config = OCIChatConfig() + result = config.transform_request( + model=_GENERIC_MODEL, + messages=[{"role": "user", "content": "hello"}], + optional_params=self._base_params, + litellm_params={}, + headers={}, + ) + assert result["compartmentId"] == _MANUAL_CREDS["oci_compartment_id"] + chat_req = result["chatRequest"] + assert chat_req["apiFormat"] == "GENERIC" + + def test_cohere_model_transform(self): + config = OCIChatConfig() + result = config.transform_request( + model=_COHERE_MODEL, + messages=[{"role": "user", "content": "tell me a joke"}], + optional_params=self._base_params, + litellm_params={}, + headers={}, + ) + chat_req = result["chatRequest"] + assert chat_req["apiFormat"] == "COHERE" + assert chat_req["message"] == "tell me a joke" + + def test_cohere_model_with_system_preamble(self): + config = OCIChatConfig() + result = config.transform_request( + model=_COHERE_MODEL, + messages=[ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "hi"}, + ], + optional_params=self._base_params, + litellm_params={}, + headers={}, + ) + assert result["chatRequest"]["preambleOverride"] == "You are helpful." + + def test_raises_without_compartment_id(self): + config = OCIChatConfig() + params = {k: v for k, v in _MANUAL_CREDS.items() if k != "oci_compartment_id"} + with pytest.raises(OCIError) as exc_info: + config.transform_request( + model=_GENERIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + optional_params=params, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert "oci_compartment_id" in str(exc_info.value) + + +# =========================================================================== +# transformation.py — OCIStreamWrapper.chunk_creator +# =========================================================================== + + +class TestOCIStreamWrapperChunkCreator: + def _make_wrapper(self, model: str) -> "OCIStreamWrapper": + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + return OCIStreamWrapper( + completion_stream=iter([]), + model=model, + custom_llm_provider="oci", + logging_obj=MagicMock(), + ) + + def test_cohere_chunk_dispatched_correctly(self): + wrapper = self._make_wrapper(_COHERE_MODEL) + payload = json.dumps( + {"apiFormat": "COHERE", "text": "hi", "finishReason": None} + ) + result = wrapper.chunk_creator(f"data:{payload}") + assert result.choices[0].delta.content == "hi" + + def test_generic_chunk_dispatched_correctly(self): + wrapper = self._make_wrapper(_GENERIC_MODEL) + payload = json.dumps( + { + "finishReason": "COMPLETE", + "index": 0, + } + ) + result = wrapper.chunk_creator(f"data:{payload}") + assert result.choices[0].finish_reason == "stop" + + def test_raises_on_non_data_prefix(self): + wrapper = self._make_wrapper(_GENERIC_MODEL) + with pytest.raises(ValueError, match="does not start with 'data:'"): + wrapper.chunk_creator("event: done") + + def test_raises_on_non_string_chunk(self): + wrapper = self._make_wrapper(_GENERIC_MODEL) + with pytest.raises(ValueError, match="not a string"): + wrapper.chunk_creator({"bad": "type"}) + + def test_empty_string_content_does_not_mark_text_emitted(self): + # An intermediate Cohere chunk carrying `text=""` must not flip the + # _cohere_text_emitted flag — otherwise a subsequent terminal + # consolidation chunk would have its real text suppressed as a + # "duplicate" and the response would be lost. + wrapper = self._make_wrapper(_COHERE_MODEL) + empty_payload = json.dumps( + {"apiFormat": "COHERE", "text": "", "finishReason": None} + ) + wrapper.chunk_creator(f"data:{empty_payload}") + assert wrapper._cohere_text_emitted is False + + terminal_payload = json.dumps( + { + "apiFormat": "COHERE", + "text": "Hello world", + "finishReason": "COMPLETE", + "chatHistory": [{"role": "CHATBOT", "message": "Hello world"}], + } + ) + result = wrapper.chunk_creator(f"data:{terminal_payload}") + assert result.choices[0].delta.content == "Hello world" + + +# =========================================================================== +# transformation.py — get_sync_custom_stream_wrapper +# =========================================================================== + + +def test_get_sync_custom_stream_wrapper_returns_wrapper(): + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + config = OCIChatConfig() + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_text.return_value = iter( + ['data:{"finishReason":"COMPLETE","index":0}'] + ) + + mock_client = MagicMock() + mock_client.post.return_value = mock_response + + wrapper = config.get_sync_custom_stream_wrapper( + model=_GENERIC_MODEL, + custom_llm_provider="oci", + logging_obj=MagicMock(), + api_base=_API_BASE, + headers={"authorization": "Signature ..."}, + data={"chatRequest": {}}, + messages=[{"role": "user", "content": "hi"}], + client=mock_client, + signed_json_body=b'{"chatRequest":{}}', + ) + + assert isinstance(wrapper, OCIStreamWrapper) + mock_client.post.assert_called_once() + + +@pytest.mark.asyncio +async def test_get_async_custom_stream_wrapper_returns_wrapper(): + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + config = OCIChatConfig() + + async def _fake_aiter_text(): + yield 'data:{"finishReason":"COMPLETE","index":0}' + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aiter_text = _fake_aiter_text + + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + + wrapper = await config.get_async_custom_stream_wrapper( + model=_GENERIC_MODEL, + custom_llm_provider="oci", + logging_obj=MagicMock(), + api_base=_API_BASE, + headers={"authorization": "Signature ..."}, + data={"chatRequest": {}}, + messages=[{"role": "user", "content": "hi"}], + client=mock_client, + signed_json_body=b'{"chatRequest":{}}', + ) + + assert isinstance(wrapper, OCIStreamWrapper) From 3b2ce201d8fce4b8b3dca2bd07cfdf122995a1a8 Mon Sep 17 00:00:00 2001 From: Michael-RZ-Berri Date: Sat, 23 May 2026 12:15:44 -0700 Subject: [PATCH 002/109] encrypt callback_vars in key/team metadata at rest (#27141) Co-authored-by: Michael Riad Zaky Co-authored-by: Yuneng Jiang --- litellm/proxy/common_utils/callback_utils.py | 99 +++++++++++- litellm/proxy/litellm_pre_call_utils.py | 7 +- .../key_management_endpoints.py | 9 +- .../team_callback_endpoints.py | 3 + .../management_endpoints/team_endpoints.py | 9 ++ tests/proxy_unit_tests/test_proxy_utils.py | 7 +- .../proxy/common_utils/test_callback_utils.py | 143 ++++++++++++++++++ .../test_key_management_endpoints.py | 33 ++++ .../test_team_callback_endpoints.py | 39 +++++ .../test_team_endpoints.py | 65 ++++++++ .../proxy/test_litellm_pre_call_utils.py | 51 +++++++ 11 files changed, 456 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 198d9503cb0a..4995752d4416 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -1,16 +1,31 @@ -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Literal, Optional +import copy +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Optional import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) from litellm.proxy.types_utils.utils import get_instance_fn from litellm.types.utils import ( StandardLoggingGuardrailInformation, StandardLoggingPayload, ) +_CALLBACK_VAR_MASKER = SensitiveDataMasker() +# Compound names that are credential-bearing but don't contain any of the +# default sensitive segments (so SensitiveDataMasker won't flag them). +_EXTRA_SENSITIVE_CALLBACK_KEYS = {"gcs_path_service_account"} +# Sentinel prefix on encrypted callback_var values. Lets us detect +# already-encrypted input cheaply (no decrypt-attempt round trip) and +# avoid double-encrypting if `LITELLM_SALT_KEY` is rotated between writes. +_CALLBACK_VAR_ENCRYPTED_PREFIX = "litellm_enc::" + blue_color_code = "\033[94m" reset_color_code = "\033[0m" @@ -550,3 +565,85 @@ def normalize_callback_names(callbacks: Iterable[Any]) -> List[Any]: if callbacks is None: return [] return [c.lower() if isinstance(c, str) else c for c in callbacks] + + +def encrypt_callback_vars(metadata: Any) -> Any: + """Return a deep copy of metadata with callback_vars values encrypted at rest. + + Idempotent: a value that already decrypts cleanly is left unchanged so + round-trips through edit forms don't double-encrypt. + """ + return _transform_callback_vars(metadata, _encrypt_if_plaintext) + + +def decrypt_callback_vars(metadata: Any) -> Any: + """Return a deep copy of metadata with callback_vars values decrypted. + + Legacy plaintext rows pass through unchanged (decrypt failure → original). + """ + return _transform_callback_vars(metadata, _decrypt_or_passthrough) + + +def _transform_callback_vars( + metadata: Any, transform: Callable[[str, Any], Any] +) -> Any: + if not isinstance(metadata, dict): + return metadata + out = copy.deepcopy(metadata) + logging_entries = out.get("logging") + if isinstance(logging_entries, list): + for entry in logging_entries: + if isinstance(entry, dict) and isinstance(entry.get("callback_vars"), dict): + entry["callback_vars"] = { + k: transform(k, v) for k, v in entry["callback_vars"].items() + } + callback_settings = out.get("callback_settings") + if isinstance(callback_settings, dict) and isinstance( + callback_settings.get("callback_vars"), dict + ): + callback_settings["callback_vars"] = { + k: transform(k, v) for k, v in callback_settings["callback_vars"].items() + } + return out + + +def _is_sensitive_callback_var(key: str) -> bool: + """Match codebase precedent: only credential-bearing fields get encrypted; + routing/identifier fields (host, base_url, project, region) stay plain.""" + if key in _EXTRA_SENSITIVE_CALLBACK_KEYS: + return True + return _CALLBACK_VAR_MASKER.is_sensitive_key(key) + + +def _encrypt_if_plaintext(key: str, value: Any) -> Any: + if not isinstance(value, str) or not value: + return value + if not _is_sensitive_callback_var(key): + return value + if value.startswith(_CALLBACK_VAR_ENCRYPTED_PREFIX): + # Already encrypted — round-tripping ciphertext (e.g. UI Edit Settings + # save without changing the field) must not double-encrypt. Cheap + # prefix check is robust under salt-key rotation; a decrypt-based + # idempotency check would mis-classify K1-encrypted blobs as + # plaintext under K2 and wrap them a second time. + return value + try: + return _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value) + except Exception: + # No salt key / master key configured — leave the value as-is rather + # than crash the write. Dev environments without LITELLM_SALT_KEY hit + # this path; production always has a master key so encryption proceeds. + return value + + +def _decrypt_or_passthrough(key: str, value: Any) -> Any: + if not isinstance(value, str) or not value: + return value + if not value.startswith(_CALLBACK_VAR_ENCRYPTED_PREFIX): + # Legacy plaintext rows or non-credential fields — return as-is. + return value + inner = value[len(_CALLBACK_VAR_ENCRYPTED_PREFIX) :] + decrypted = decrypt_value_helper( + value=inner, key=key, exception_type="debug", return_original_value=False + ) + return decrypted if decrypted is not None else value diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8cb9a11ffeed..2b840b5495e2 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -26,6 +26,7 @@ UserAPIKeyAuth, ) from litellm.proxy.common_utils.callback_utils import ( + decrypt_callback_vars, get_metadata_variable_name_from_kwargs, ) from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers @@ -477,7 +478,7 @@ def get_key_dynamic_logging_settings(user_api_key_dict: UserAPIKeyAuth): user_api_key_dict.metadata is not None and "logging" in user_api_key_dict.metadata ): - return user_api_key_dict.metadata["logging"] + return decrypt_callback_vars(user_api_key_dict.metadata).get("logging") return None @staticmethod @@ -486,7 +487,7 @@ def get_team_dynamic_logging_settings(user_api_key_dict: UserAPIKeyAuth): user_api_key_dict.team_metadata is not None and "logging" in user_api_key_dict.team_metadata ): - return user_api_key_dict.team_metadata["logging"] + return decrypt_callback_vars(user_api_key_dict.team_metadata).get("logging") return None @@ -540,7 +541,7 @@ def _get_dynamic_logging_metadata( } } """ - team_metadata = user_api_key_dict.team_metadata + team_metadata = decrypt_callback_vars(user_api_key_dict.team_metadata) callback_settings = team_metadata.get("callback_settings", None) or {} callback_settings_obj = TeamCallbackMetadata(**callback_settings) verbose_proxy_logger.debug( diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2ab147043d9e..eebcf0b65714 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -51,6 +51,10 @@ ) from litellm.proxy.auth.auth_utils import abbreviate_api_key from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.callback_utils import ( + decrypt_callback_vars, + encrypt_callback_vars, +) from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks @@ -1752,7 +1756,7 @@ def prepare_metadata_fields( ) ) - non_default_values["metadata"] = casted_metadata + non_default_values["metadata"] = encrypt_callback_vars(casted_metadata) return non_default_values @@ -3459,6 +3463,7 @@ async def generate_key_helper_fn( # noqa: PLR0915 metadata = metadata or {} metadata["prompts"] = prompts + metadata = encrypt_callback_vars(metadata) metadata_json = json.dumps(metadata) validate_model_max_budget(model_max_budget) model_max_budget_json = json.dumps(model_max_budget) @@ -5942,7 +5947,7 @@ async def key_health( logging_statuses = await test_key_logging( user_api_key_dict=user_api_key_dict, request=request, - key_logging=key_metadata["logging"], + key_logging=decrypt_callback_vars(key_metadata)["logging"], ) health_status["logging_callbacks"] = logging_statuses diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 17d86410a0fc..63b56425b0ed 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -27,6 +27,7 @@ UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.management_endpoints.team_endpoints import _verify_team_access from litellm.proxy.management_helpers.utils import management_endpoint_wrapper @@ -245,6 +246,7 @@ async def add_team_callbacks( team_callback_settings.append(data.model_dump()) team_metadata["logging"] = team_callback_settings + team_metadata = encrypt_callback_vars(team_metadata) team_metadata_json = json.dumps(team_metadata) # update team_metadata new_team_row = await prisma_client.db.litellm_teamtable.update( @@ -347,6 +349,7 @@ async def disable_team_logging( # Update metadata team_metadata["callback_settings"] = team_callback_settings_obj.model_dump() + team_metadata = encrypt_callback_vars(team_metadata) team_metadata_json = json.dumps(team_metadata) # Update team in database diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0b2f93d817a8..43fdc9ae1cf2 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -72,6 +72,7 @@ get_user_object, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.management_endpoints.common_utils import ( _check_passthrough_routes_caller_permission, _is_user_org_admin_for_team, @@ -1156,6 +1157,11 @@ async def new_team( # noqa: PLR0915 ) complete_team_data_dict["router_settings"] = router_settings_json + if complete_team_data_dict.get("metadata") is not None: + complete_team_data_dict["metadata"] = encrypt_callback_vars( + complete_team_data_dict["metadata"] + ) + complete_team_data_dict = prisma_client.jsonify_team_object( db_data=complete_team_data_dict ) @@ -1828,6 +1834,9 @@ async def update_team( # noqa: PLR0915 # update team metadata fields _update_metadata_fields(updated_kv=updated_kv) + if updated_kv.get("metadata") is not None: + updated_kv["metadata"] = encrypt_callback_vars(updated_kv["metadata"]) + if "model_aliases" in updated_kv: updated_kv.pop("model_aliases") _model_id = await _update_model_table( diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index ff695d4a57ef..d36d73da2c3a 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -416,9 +416,10 @@ def test_dynamic_turn_off_message_logging(callback_vars): ) assert callbacks is not None - assert ( - callbacks.callback_vars["turn_off_message_logging"] - == callback_vars["turn_off_message_logging"] + # AddTeamCallback's validator stringifies callback_var values, so compare + # against the str() of the input rather than the input bool directly. + assert callbacks.callback_vars["turn_off_message_logging"] == str( + callback_vars["turn_off_message_logging"] ) diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index c6132194c74b..cb30970a34e7 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -1,3 +1,4 @@ +import copy import sys import os from types import SimpleNamespace @@ -7,6 +8,8 @@ ) # Adds the parent directory to the system path from litellm.proxy.common_utils.callback_utils import ( + decrypt_callback_vars, + encrypt_callback_vars, initialize_callbacks_on_proxy, get_remaining_tokens_and_requests_from_request_data, normalize_callback_names, @@ -119,3 +122,143 @@ def test_initialize_callbacks_on_proxy_instantiates_compression_interception( assert "compression_interception" not in litellm.callbacks finally: litellm.callbacks = original_callbacks + + +# --------------------------------------------------------------------------- +# encrypt_callback_vars / decrypt_callback_vars +# --------------------------------------------------------------------------- + + +def _sample_metadata(): + return { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": { + "langfuse_public_key": "pk-lf-public", + "langfuse_secret_key": "sk-lf-secret", + "langfuse_host": "https://cloud.langfuse.com", + }, + } + ], + "callback_settings": { + "callback_vars": {"langsmith_api_key": "ls-api-key"}, + }, + "tags": ["unrelated"], + } + + +def _set_salt_key(monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + + +def test_encrypt_callback_vars_round_trip(monkeypatch): + _set_salt_key(monkeypatch) + original = _sample_metadata() + encrypted = encrypt_callback_vars(original) + + enc_vars = encrypted["logging"][0]["callback_vars"] + assert enc_vars["langfuse_secret_key"] != "sk-lf-secret" + assert enc_vars["langfuse_public_key"] != "pk-lf-public" + assert ( + encrypted["callback_settings"]["callback_vars"]["langsmith_api_key"] + != "ls-api-key" + ) + + decrypted = decrypt_callback_vars(encrypted) + assert ( + decrypted["logging"][0]["callback_vars"] + == original["logging"][0]["callback_vars"] + ) + assert ( + decrypted["callback_settings"]["callback_vars"] + == original["callback_settings"]["callback_vars"] + ) + + +def test_encrypt_callback_vars_is_idempotent(monkeypatch): + _set_salt_key(monkeypatch) + once = encrypt_callback_vars(_sample_metadata()) + twice = encrypt_callback_vars(once) + assert once == twice + + +def test_encrypt_callback_vars_does_not_mutate_input(monkeypatch): + _set_salt_key(monkeypatch) + original = _sample_metadata() + snapshot = copy.deepcopy(original) + encrypt_callback_vars(original) + assert original == snapshot + + +def test_decrypt_callback_vars_passes_through_legacy_plaintext(monkeypatch): + _set_salt_key(monkeypatch) + plaintext = _sample_metadata() + decrypted = decrypt_callback_vars(plaintext) + # legacy rows decrypt-fail and fall through unchanged + assert ( + decrypted["logging"][0]["callback_vars"]["langfuse_secret_key"] + == "sk-lf-secret" + ) + + +def test_callback_vars_helpers_handle_edge_shapes(monkeypatch): + _set_salt_key(monkeypatch) + assert encrypt_callback_vars(None) is None + assert encrypt_callback_vars({}) == {} + assert decrypt_callback_vars(None) is None + assert decrypt_callback_vars({}) == {} + + # logging not a list / callback_vars not a dict — leave alone + weird = {"logging": "not-a-list", "callback_settings": {"callback_vars": None}} + assert encrypt_callback_vars(weird) == weird + + # empty/None callback_vars values stay as-is + has_blanks = { + "logging": [ + { + "callback_vars": { + "langfuse_public_key": "", + "langfuse_secret_key": None, + "langfuse_host": "https://cloud.langfuse.com", + } + } + ] + } + out = encrypt_callback_vars(has_blanks) + cv = out["logging"][0]["callback_vars"] + assert cv["langfuse_public_key"] == "" + assert cv["langfuse_secret_key"] is None + # langfuse_host is a routing field, not a credential — stays plain. + assert cv["langfuse_host"] == "https://cloud.langfuse.com" + + +def test_encrypt_callback_vars_only_encrypts_credential_fields(monkeypatch): + """Routing/identifier fields stay plaintext; credential fields encrypt.""" + _set_salt_key(monkeypatch) + metadata = { + "logging": [ + { + "callback_vars": { + "langfuse_secret_key": "sk-real", + "langfuse_public_key": "pk-real", + "langfuse_host": "https://cloud.langfuse.com", + "langsmith_project": "my-proj", + "langsmith_base_url": "https://smith.example", + "gcs_path_service_account": "{json contents}", + } + } + ] + } + cv = encrypt_callback_vars(metadata)["logging"][0]["callback_vars"] + + # Sensitive (key-name segments match SensitiveDataMasker patterns): + assert cv["langfuse_secret_key"] != "sk-real" + assert cv["langfuse_public_key"] != "pk-real" + # Sensitive via the explicit gcs override: + assert cv["gcs_path_service_account"] != "{json contents}" + # Routing / identifiers stay plaintext: + assert cv["langfuse_host"] == "https://cloud.langfuse.com" + assert cv["langsmith_project"] == "my-proj" + assert cv["langsmith_base_url"] == "https://smith.example" diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index c2ac941e7370..38bf2d2c915d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1336,6 +1336,39 @@ async def test_update_without_metadata_still_preserves_existing(): assert result["metadata"]["other"] == "kept" +@pytest.mark.asyncio +async def test_prepare_key_update_data_encrypts_callback_vars(monkeypatch): + """/key/update must encrypt callback_vars values before they reach the DB.""" + from litellm.proxy.common_utils.callback_utils import decrypt_callback_vars + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + data = UpdateKeyRequest( + key="sk-1", + metadata={ + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-real", + "langfuse_secret_key": "sk-real", + }, + } + ] + }, + ) + existing_key = LiteLLM_VerificationToken(token="hashed") + + result = await prepare_key_update_data(data=data, existing_key_row=existing_key) + + cv = result["metadata"]["logging"][0]["callback_vars"] + assert cv["langfuse_secret_key"] != "sk-real" + assert cv["langfuse_public_key"] != "pk-real" + recovered = decrypt_callback_vars(result["metadata"])["logging"][0]["callback_vars"] + assert recovered["langfuse_secret_key"] == "sk-real" + assert recovered["langfuse_public_key"] == "pk-real" + + @pytest.mark.asyncio async def test_prepare_key_update_data_duration_never_expires(): """Test that duration="-1" sets expires to None (never expires).""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index 55afd4061d87..d43bf3a3bd8e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -420,3 +420,42 @@ async def capture(request_data): ) assert audit_calls == [] + + +@pytest.mark.asyncio +async def test_add_team_callbacks_writes_encrypted_callback_vars(monkeypatch): + """add_team_callbacks must encrypt callback_vars values before the DB write.""" + from litellm.proxy.common_utils.callback_utils import decrypt_callback_vars + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata={"logging": []})) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch("litellm.proxy.proxy_server.master_key", None), + ): + await add_team_callbacks( + data=AddTeamCallback( + callback_name="langfuse", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk-lf-real-public", + "langfuse_secret_key": "sk-lf-real-secret", + }, + ), + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + written = json.loads( + mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"] + ) + cv = written["logging"][0]["callback_vars"] + assert cv["langfuse_secret_key"] != "sk-lf-real-secret" + assert cv["langfuse_public_key"] != "pk-lf-real-public" + recovered = decrypt_callback_vars(written)["logging"][0]["callback_vars"] + assert recovered["langfuse_secret_key"] == "sk-lf-real-secret" + assert recovered["langfuse_public_key"] == "pk-lf-real-public" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index b450a2629074..41a5b891ad39 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -7945,6 +7945,71 @@ async def test_team_member_me_returns_404_for_unknown_team(mock_db_client): assert exc_info.value.status_code == 404 +@pytest.mark.asyncio +async def test_new_team_encrypts_callback_vars( + mock_db_client, mock_admin_auth, monkeypatch +): + """/team/new must encrypt callback_vars values before they reach the DB.""" + from fastapi import Request + + from litellm.proxy._types import NewTeamRequest + from litellm.proxy.common_utils.callback_utils import decrypt_callback_vars + from litellm.proxy.management_endpoints.team_endpoints import new_team + from litellm.proxy.utils import PrismaClient + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + + # Use the real jsonify helpers so the encrypted dict goes through the + # actual JSON serialization production uses (catches non-serializable + # ciphertext, missing fields, etc.). + mock_db_client.jsonify_object = PrismaClient.jsonify_object.__get__(mock_db_client) + mock_db_client.jsonify_team_object = PrismaClient.jsonify_team_object.__get__( + mock_db_client + ) + mock_db_client.get_data = AsyncMock(return_value=None) + mock_db_client.db = MagicMock() + mock_db_client.db.litellm_teamtable = MagicMock() + team_create_result = MagicMock(team_id="team-456", object_permission_id=None) + team_create_result.model_dump.return_value = {"team_id": "team-456"} + mock_team_create = AsyncMock(return_value=team_create_result) + mock_db_client.db.litellm_teamtable.create = mock_team_create + mock_db_client.db.litellm_teamtable.count = AsyncMock(return_value=0) + mock_db_client.db.litellm_teamtable.update = AsyncMock( + return_value=team_create_result + ) + mock_db_client.db.litellm_usertable = MagicMock() + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + team_request = NewTeamRequest( + team_alias="my-team", + metadata={ + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-real", + "langfuse_secret_key": "sk-real", + }, + } + ] + }, + ) + + await new_team( + data=team_request, + http_request=MagicMock(spec=Request), + user_api_key_dict=mock_admin_auth, + ) + + written = mock_team_create.call_args.kwargs["data"] + # jsonify_team_object serializes the metadata dict to a JSON string before + # the DB write, so we round-trip through json.loads to inspect it. + metadata = json.loads(written["metadata"]) + cv = metadata["logging"][0]["callback_vars"] + assert cv["langfuse_secret_key"] != "sk-real" + recovered = decrypt_callback_vars(metadata)["logging"][0]["callback_vars"] + assert recovered["langfuse_secret_key"] == "sk-real" def _non_admin_auth(): return UserAPIKeyAuth( user_id="u-team-admin", user_role=LitellmUserRoles.INTERNAL_USER diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 19204e8b8ab0..fc9813ba5302 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -1549,6 +1549,57 @@ def test_team_dynamic_logging_settings(): assert result is None +def test_key_dynamic_logging_settings_decrypts_callback_vars(monkeypatch): + """Encrypted callback_vars on the key are decrypted before downstream use.""" + from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + encrypted_metadata = encrypt_callback_vars( + { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-real", + "langfuse_secret_key": "sk-real", + }, + } + ] + } + ) + cv_on_disk = encrypted_metadata["logging"][0]["callback_vars"] + assert cv_on_disk["langfuse_secret_key"] != "sk-real" # sanity: stored encrypted + + key = UserAPIKeyAuth(api_key="t", metadata=encrypted_metadata, team_metadata={}) + result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key) + cv = result[0]["callback_vars"] + assert cv["langfuse_secret_key"] == "sk-real" + assert cv["langfuse_public_key"] == "pk-real" + + +def test_team_dynamic_logging_settings_decrypts_callback_vars(monkeypatch): + """Encrypted callback_vars on the team are decrypted before downstream use.""" + from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + encrypted_team = encrypt_callback_vars( + { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "failure", + "callback_vars": {"langfuse_secret_key": "team-sk-real"}, + } + ] + } + ) + + key = UserAPIKeyAuth(api_key="t", metadata={}, team_metadata=encrypted_team) + result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key) + assert result[0]["callback_vars"]["langfuse_secret_key"] == "team-sk-real" + + def test_get_dynamic_logging_metadata_with_arize_team_logging(): """ Test _get_dynamic_logging_metadata function with arize team logging and dynamic parameters From 2eab9ee2c0caf66b6ed51c3f3cb9b41d59cd1001 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 23 May 2026 12:15:59 -0700 Subject: [PATCH 003/109] perf: reduce per-request and per-chunk overhead across Anthropic streaming hot paths (#28289) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: reduce per-request and per-chunk overhead across Anthropic streaming hot paths - Introduce pure-text fast-path in `_build_complete_streaming_response` that collapses O(N) `content_block_delta` events into a single equivalent SSE event before conversion, eliminating per-output-token Pydantic `ModelResponseStream` construction; non-text streams (tool_use, thinking, citations) fall back to the unchanged legacy path - Skip agentic streaming wrapper entirely when no callback overrides `async_should_run_agentic_loop`; the wrapper buffered every chunk and rebuilt the SSE response only to call hooks that all return `(False, {})` — a pure no-op for the default config - Serialize request body once (`json.dumps`) for both the pre-call log input and the wire, instead of twice; avoids a full O(payload) scan per request, significant for long-context Claude Code histories - Add fast path in `async_streaming_data_generator` that bypasses the per-chunk `async_post_call_streaming_hook` coroutine await, response-string materialization, and cost-injection call when no callback/guardrail/cost-injection is active (the default config) - Resolve `_DD_STREAMING_TRACE_ENABLED` once at import time; eliminate per-chunk `NullSpan` context manager allocation when Datadog tracing is disabled (the default) - Memoize `get_type_hints(AnthropicMessagesRequestOptionalParams)` with `@lru_cache(maxsize=1)` — resolves once per process instead of once per `/v1/messages` request (~80µs each) - Hoist `cost_injection_active` out of the per-chunk loop in `chunk_processor`; eliminates repeated `getattr` + endpoint-type checks on every streamed byte chunk - Extract `_build_passthrough_logging_result` from `_route_streaming_logging_to_handler` as a standalone static method to facilitate future off-loop dispatch - Convert `async_sse_data_generator` from an `async for: yield` trampoline to a direct return of the underlying generator, removing one async-generator layer per streamed chunk - Skip redundant `strip_empty_text_blocks_from_anthropic_messages` scan in `anthropic_messages_handler` when the async wrapper already sanitized (signalled via `_litellm_messages_presanitized` sentinel, popped before reaching provider params) - Gate debug log `f-string` evaluation behind `isEnabledFor(DEBUG)` in both the streaming generator and the transformation layer to avoid serializing entire message payloads on every request at non-debug log levels - Add benchmark script (`scripts/benchmark_anthropic_messages_perf.py`) with a local mock Anthropic SSE provider for reproducible TTFT and TPM measurement across commits/branches - Add parity tests asserting fast-path and legacy-path produce byte-identical logged/billed payloads, plus unit tests for agentic hook detection, pre-serialized body reuse, and memoized key resolution * perf: address greptile review for anthropic streaming hot path - Bail to legacy in `_collapse_pure_text_chunks` when content_block_delta events from different block indexes are observed without an intervening flush. Anthropic sends blocks strictly sequentially, but defensive bail prevents silent text-merging if the protocol ever interleaves. - Replace leaf-class `__dict__` check for `async_post_call_streaming_hook` in `_callback_capabilities` with a function-identity comparison that walks the MRO. A vendor base class can carry the override and the registered class can add nothing else; before this PR the hook was unconditionally invoked, so an inherited-override miss would silently drop the hook on the streaming path. - Add unit tests for both behaviors. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(mypy): narrow model_name to str in cost-injection branch The hoisted cost_injection_active flag in chunk_processor encodes the `bool(model_name)` requirement but mypy can't track that invariant through the local, so the per-chunk `_process_chunk_with_cost_injection( chunk, model_name)` calls flagged Optional[str] vs str. Pin a typed non-None local inside the cost-injection branch so mypy narrows correctly without changing runtime behavior. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Yassin Kortam Co-authored-by: Claude Opus 4.7 (1M context) --- .../messages/handler.py | 16 +- .../messages/transformation.py | 5 +- .../messages/utils.py | 17 +- litellm/llms/custom_httpx/llm_http_handler.py | 73 +- litellm/proxy/common_request_processing.py | 62 +- .../anthropic_passthrough_logging_handler.py | 169 ++++- .../streaming_handler.py | 191 ++++-- litellm/proxy/utils.py | 17 +- scripts/benchmark_anthropic_messages_perf.py | 624 ++++++++++++++++++ ...erimental_pass_through_messages_handler.py | 129 ++++ .../test_request_optional_param_utils.py | 56 ++ .../custom_httpx/test_llm_http_handler.py | 207 ++++++ ...t_anthropic_passthrough_logging_handler.py | 359 ++++++++++ .../proxy/test_common_request_processing.py | 122 +++- .../test_proxy_logging_hook_detection.py | 22 + 15 files changed, 1978 insertions(+), 91 deletions(-) create mode 100644 scripts/benchmark_anthropic_messages_perf.py create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 009ba6ef306d..14e06e047eab 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -293,6 +293,12 @@ async def anthropic_messages( api_base=api_base, client=client, custom_llm_provider=custom_llm_provider, + # messages were already empty-text-block sanitized at the top of this + # function and are NOT reassigned before this dispatch, so the handler + # can skip its (otherwise redundant) second full-messages scan. Passed + # explicitly (not via **kwargs) so it only affects this direct + # dispatch -- interceptor / sync entry points still sanitize. + _litellm_messages_presanitized=True, **kwargs, ) ctx = contextvars.copy_context() @@ -351,10 +357,14 @@ def anthropic_messages_handler( """ from litellm.types.utils import LlmProviders - # Sanitize empty text blocks here too so the sync entry point + # Sanitize empty text blocks so the sync entry point # (litellm.messages.create -> anthropic_messages_handler) gets the same - # protection as the async wrapper. Idempotent when called twice. - messages = strip_empty_text_blocks_from_anthropic_messages(messages) + # protection as the async wrapper. The async wrapper already sanitized and + # does not reassign messages before dispatch, so it sets + # ``_litellm_messages_presanitized`` to skip this redundant second + # full-messages scan. Pop it so it never leaks into provider params. + if not kwargs.pop("_litellm_messages_presanitized", False): + messages = strip_empty_text_blocks_from_anthropic_messages(messages) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 35495d596107..15f404d3f530 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -312,7 +312,10 @@ def transform_anthropic_messages_request( ) ####### get required params for all anthropic messages requests ###### - verbose_logger.debug(f"TRANSFORMATION DEBUG - Messages: {messages}") + # Lazy %s: the f-string previously stringified the entire messages + # payload on every request regardless of log level (a full scan of the + # request body on the hot path). Defer it to when DEBUG is enabled. + verbose_logger.debug("TRANSFORMATION DEBUG - Messages: %s", messages) # Auto-strip advisor blocks from history if advisor tool is absent. # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index fa951ebd2e5a..88832fb3f63f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -1,4 +1,5 @@ -from typing import Any, Dict, List, cast, get_type_hints +from functools import lru_cache +from typing import Any, Dict, FrozenSet, List, cast, get_type_hints from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -6,6 +7,18 @@ ) +@lru_cache(maxsize=1) +def _anthropic_messages_optional_param_keys() -> FrozenSet[str]: + """ + Valid AnthropicMessagesRequestOptionalParams keys. + + ``typing.get_type_hints`` is ~80us/call and this TypedDict is static, so + resolving it once per process instead of once per request removes a fixed + full-pass cost from the /v1/messages request-parse path. + """ + return frozenset(get_type_hints(AnthropicMessagesRequestOptionalParams).keys()) + + class AnthropicMessagesRequestUtils: @staticmethod def get_requested_anthropic_messages_optional_param( @@ -20,7 +33,7 @@ def get_requested_anthropic_messages_optional_param( Returns: AnthropicMessagesRequestOptionalParams instance with only the valid parameters """ - valid_keys = get_type_hints(AnthropicMessagesRequestOptionalParams).keys() + valid_keys = _anthropic_messages_optional_param_keys() filtered_params = { k: v for k, v in params.items() if k in valid_keys and v is not None } diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 090aac187b3c..74e55b9debb9 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1886,7 +1886,9 @@ async def _async_post_anthropic_messages_with_http_error_retry( async_httpx_client: AsyncHTTPHandler, request_url: str, headers: dict, - signed_json_body: Optional[bytes], + # str when the caller passes a pre-serialized (unsigned) body to avoid + # re-dumping; bytes when a provider signed the request (e.g. Bedrock). + signed_json_body: Optional[Union[str, bytes]], request_body: dict, stream: bool, logging_obj: LiteLLMLoggingObj, @@ -2077,8 +2079,18 @@ async def async_anthropic_messages_handler( model=model, ) + # The request body was serialized once for the pre-call log input and + # again for the wire (json.dumps is O(payload), large for long-context + # Claude Code history). Serialize once and reuse for both. Only when + # the provider didn't sign the request (sign_request no-op for the + # native anthropic path -> signed_json_body is None); signed providers + # (e.g. Bedrock) keep their signed body untouched. The HTTP-error + # retry path mutates + re-signs the body, so it still re-serializes + # internally -- this only deduplicates the success path. + request_body_json = json.dumps(request_body) + logging_obj.pre_call( - input=[{"role": "user", "content": json.dumps(request_body)}], + input=[{"role": "user", "content": request_body_json}], api_key="", additional_args={ "complete_input_dict": request_body, @@ -2091,7 +2103,9 @@ async def async_anthropic_messages_handler( async_httpx_client=async_httpx_client, request_url=request_url, headers=headers, - signed_json_body=signed_json_body, + signed_json_body=( + signed_json_body if signed_json_body is not None else request_body_json + ), request_body=request_body, stream=stream or False, logging_obj=logging_obj, @@ -2113,6 +2127,14 @@ async def async_anthropic_messages_handler( litellm_logging_obj=logging_obj, ) + if not self._has_agentic_completion_hook(logging_obj): + # No callback overrides async_should_run_agentic_loop, so the + # agentic wrapper's only effect would be buffering every chunk + # and rebuilding the response from SSE at end-of-stream to call + # hooks that all return (False, {}). Stream through directly and + # skip that per-chunk + end-of-stream overhead. + return completion_stream + from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) @@ -4620,6 +4642,51 @@ def _get_agentic_loop_settings(kwargs: Dict) -> Tuple[int, int, List[str]]: fingerprints = list(kwargs.get("_agentic_loop_fingerprints", []) or []) return depth, max(max_loops, 1), fingerprints + @staticmethod + def _has_agentic_completion_hook(logging_obj: Any) -> bool: + """ + True if any registered callback actually overrides + ``async_should_run_agentic_loop`` (the gate every agentic hook goes + through). The base ``CustomLogger`` implementation returns + ``(False, {})``, so when nothing overrides it the agentic + post-processing is a guaranteed no-op and the streaming wrapper that + buffers + rebuilds the whole response from SSE just to call it can be + skipped entirely. + + Function-identity comparison (not a leaf ``__dict__`` check) so an + override inherited through any intermediate class is still detected -- + a false negative here would silently disable agentic features. + + String entries in ``litellm.callbacks`` (e.g. ``"datadog"``) are + resolved to their ``CustomLogger`` instance via + ``get_custom_logger_compatible_class`` -- same pattern as + ``ProxyLogging._callback_capabilities`` -- so a string-registered + agentic callback is detected too. + """ + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils.litellm_logging import ( + get_custom_logger_compatible_class, + ) + + base_func = CustomLogger.async_should_run_agentic_loop + callbacks = litellm.callbacks + ( + getattr(logging_obj, "dynamic_success_callbacks", None) or [] + ) + for cb in callbacks: + if isinstance(cb, str): + resolved = get_custom_logger_compatible_class(cb) # type: ignore[arg-type] + if resolved is None: + continue + cb = resolved + if not isinstance(cb, CustomLogger): + continue + cb_func = getattr(type(cb), "async_should_run_agentic_loop", base_func) + if getattr(cb_func, "__func__", cb_func) is not getattr( + base_func, "__func__", base_func + ): + return True + return False + @staticmethod def _check_agentic_loop_safety( tool_calls: Any, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7d2954fd2dd1..ef1d64335b40 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -32,7 +32,7 @@ STREAM_SSE_DATA_PREFIX, ) from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.dd_tracing import tracer +from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, @@ -65,6 +65,13 @@ from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.types.utils import ModelResponse, ModelResponseStream, Usage +# Datadog streaming spans are a no-op when ddtrace is not enabled, but the +# ``with tracer.trace(...)`` context manager still allocates a NullSpan and +# runs __enter__/__exit__ for every streamed chunk. Resolve once at import so +# the per-chunk hot path can skip the context manager entirely when tracing +# is off (the default). +_DD_STREAMING_TRACE_ENABLED = not isinstance(tracer, NullTracer) + def _serialize_http_exception_detail( detail: Any, @@ -231,7 +238,7 @@ def _extract_error_from_sse_chunk(event_line: Union[str, bytes]) -> dict: return default_error -async def create_response( +async def create_response( # noqa: PLR0915 generator: AsyncGenerator[str, None], media_type: str, headers: dict, @@ -336,6 +343,13 @@ async def error_gen_message() -> AsyncGenerator[str, None]: ) async def combined_generator() -> AsyncGenerator[str, None]: + if not _DD_STREAMING_TRACE_ENABLED: + # Fast path: no per-chunk span object / context-manager overhead. + if first_chunk_value is not None: + yield first_chunk_value + async for chunk in generator: + yield chunk + return if first_chunk_value is not None: with tracer.trace(DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE): yield first_chunk_value @@ -1900,6 +1914,23 @@ async def async_streaming_data_generator( failure hook and yields via serialize_error. Use for SSE or NDJSON. """ verbose_proxy_logger.debug("inside generator") + # Resolve per-stream (not per-chunk) whether the heavy per-chunk path + # is needed. When no callback overrides ``async_post_call_streaming_hook``, + # no CustomGuardrail is active, and cost injection is disabled, the + # per-chunk hook returns the chunk unchanged, ``str_so_far`` is never + # consumed, and cost injection is a no-op -- so the per-chunk coroutine + # await, response-string materialization, and cost-injection call are + # pure overhead on the streaming hot path (the default config). + caps = ProxyLogging._callback_capabilities() + cost_injection_enabled = bool( + getattr(litellm, "include_cost_in_streaming_usage", False) + ) + fast_path = ( + not caps.has_streaming_chunk_override + and not caps.has_guardrail + and not cost_injection_enabled + ) + debug_enabled = verbose_proxy_logger.isEnabledFor(logging.DEBUG) try: str_so_far = "" async for ( @@ -1909,9 +1940,17 @@ async def async_streaming_data_generator( response=response, request_data=request_data, ): - verbose_proxy_logger.debug( - "async_data_generator: received streaming chunk - {}".format(chunk) - ) + # ``.format(chunk)`` was previously evaluated for every chunk + # regardless of log level; gate it behind the level check. + if debug_enabled: + verbose_proxy_logger.debug( + "async_data_generator: received streaming chunk - %s", chunk + ) + + if fast_path: + yield serialize_chunk(chunk) + continue + chunk = await proxy_logging_obj.async_post_call_streaming_hook( user_api_key_dict=user_api_key_dict, response=chunk, @@ -1969,7 +2008,7 @@ async def async_streaming_data_generator( yield serialize_error(proxy_exception) @staticmethod - async def async_sse_data_generator( + def async_sse_data_generator( response: Any, user_api_key_dict: UserAPIKeyAuth, request_data: dict, @@ -1977,17 +2016,20 @@ async def async_sse_data_generator( ) -> AsyncGenerator[str, None]: """ Anthropic /messages and Google /generateContent streaming data generator require SSE events. - Delegates to async_streaming_data_generator with SSE serializers. + + Returns the underlying ``async_streaming_data_generator`` configured with + SSE serializers directly (rather than re-wrapping it in another + ``async for: yield`` trampoline), so a streamed chunk traverses one + fewer async-generator layer / coroutine resume on the hot path. """ - async for chunk in ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + return ProxyBaseLLMRequestProcessing.async_streaming_data_generator( response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, proxy_logging_obj=proxy_logging_obj, serialize_chunk=ProxyBaseLLMRequestProcessing.return_sse_chunk, serialize_error=lambda proxy_exc: f"{STREAM_SSE_DATA_PREFIX}{json.dumps({'error': proxy_exc.to_dict()})}\n\n", - ): - yield chunk + ) @staticmethod def _process_chunk_with_cost_injection(chunk: Any, model_name: str) -> Any: diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c42faa59cf04..3be26eb572dd 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -266,7 +266,174 @@ def _build_complete_streaming_response( model: str, ) -> Optional[Union[ModelResponse, TextCompletionResponse]]: """ - Builds complete response from raw Anthropic chunks + Builds complete response from raw Anthropic chunks. + + Fast path: for the dominant case of a pure-text streaming response + (no tool_use / thinking / non-text content blocks), the long run of + ``content_block_delta`` text deltas is collapsed into a single + equivalent SSE event before conversion. ``chunk_parser`` and + ``stream_chunk_builder`` remain the single source of truth for chunk + shape, usage math and finish-reason mapping, so the rebuilt response + (and therefore the logged/billed payload) is identical -- this is + asserted by a parity test. Anything non-trivial falls back to the + unchanged legacy reconstruction. + + Per-event Pydantic ``ModelResponseStream`` construction dominated + event-loop CPU under concurrent streaming; collapsing the homogeneous + text run removes O(num_output_tokens) of it. + """ + collapsed = AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks( + all_chunks + ) + if collapsed is not None: + return AnthropicPassthroughLoggingHandler._build_complete_streaming_response_legacy( + all_chunks=collapsed, + litellm_logging_obj=litellm_logging_obj, + model=model, + ) + return AnthropicPassthroughLoggingHandler._build_complete_streaming_response_legacy( + all_chunks=all_chunks, + litellm_logging_obj=litellm_logging_obj, + model=model, + ) + + # Anthropic SSE block/delta types that the fast path is NOT allowed to + # collapse -- their presence forces the unchanged legacy path so tool + # calls, thinking, citations, etc. keep byte-identical reconstruction. + _FAST_PATH_DISALLOWED_DELTA_TYPES = frozenset( + { + "input_json_delta", + "thinking_delta", + "signature_delta", + "citations_delta", + } + ) + + @staticmethod + def _collapse_pure_text_chunks( # noqa: PLR0915 + all_chunks: Sequence[Union[str, bytes]], + ) -> Optional[List[str]]: + """ + Return a new chunk list with the contiguous run of text-only + ``content_block_delta`` events replaced by a single equivalent event, + or ``None`` if the stream is not a pure single-text-block response + (in which case the caller uses the legacy path unchanged). + + Only ``message_start`` / ``content_block_start(text)`` / + ``content_block_delta(text_delta)`` / ``content_block_stop`` / + ``message_delta`` / ``message_stop`` / ``ping`` events are accepted. + Any other content-block type or delta type returns ``None``. + """ + normalized: List[str] = [] + for raw in all_chunks: + line = raw.decode("utf-8") if isinstance(raw, bytes) else raw + for ev in line.split("\n\n"): + ev = ev.strip() + if ev: + normalized.append(ev) + + text_block_indexes: set = set() + out: List[str] = [] + pending_text: List[str] = [] + pending_index: Optional[int] = None + saw_any_text_delta = False + + def flush() -> None: + nonlocal pending_text, pending_index + if pending_text: + merged = { + "type": "content_block_delta", + "index": pending_index if pending_index is not None else 0, + "delta": {"type": "text_delta", "text": "".join(pending_text)}, + } + out.append("data: " + json.dumps(merged)) + pending_text = [] + pending_index = None + + for ev in normalized: + idx = ev.find("data:") + if idx == -1: + # Bare "event: " line. The legacy converter turns this + # into an empty ModelResponseStream that contributes nothing + # to stream_chunk_builder. Drop the high-frequency interior + # markers (content_block_delta / ping); keep every other + # bare event line verbatim so chunk ordering and the + # load-bearing chunks[0] (event: message_start) are retained. + name = ev[len("event:") :].strip() if ev.startswith("event:") else "" + if name in ("content_block_delta", "ping"): + continue + flush() + out.append(ev) + continue + + json_str = ev[idx + len("data:") :].strip() + try: + data = json.loads(json_str) + except (json.JSONDecodeError, ValueError): + return None + + etype = data.get("type") + if etype == "content_block_start": + block = data.get("content_block") or {} + if block.get("type") != "text": + return None + text_block_indexes.add(data.get("index")) + flush() + out.append(ev) + elif etype == "content_block_delta": + delta = data.get("delta") or {} + dtype = delta.get("type") + if ( + dtype + in AnthropicPassthroughLoggingHandler._FAST_PATH_DISALLOWED_DELTA_TYPES + ): + return None + if dtype != "text_delta": + return None + cur_index = data.get("index") + if cur_index not in text_block_indexes: + return None + # Defensive: Anthropic sends blocks strictly sequentially + # (start/deltas/stop, then next block), so pending_text from + # block N must be flushed by content_block_stop before block + # N+1's deltas arrive. If we ever see a delta whose index + # disagrees with the current pending buffer, the stream is + # interleaved -- fall back to legacy rather than risk merging + # text from different blocks under a single index. + if ( + pending_text + and pending_index is not None + and cur_index != pending_index + ): + return None + saw_any_text_delta = True + pending_index = cur_index + pending_text.append(delta.get("text") or "") + elif etype == "ping": + # Interior no-op; legacy maps it to an empty chunk. + continue + else: + # message_start / content_block_stop / message_delta / + # message_stop / error: pass through unchanged. + flush() + out.append(ev) + + flush() + + if not saw_any_text_delta: + return None + return out + + @staticmethod + def _build_complete_streaming_response_legacy( + all_chunks: Sequence[Union[str, bytes]], + litellm_logging_obj: LiteLLMLoggingObj, + model: str, + ) -> Optional[Union[ModelResponse, TextCompletionResponse]]: + """ + Original reconstruction: convert every SSE event to a generic chunk + and assemble via stream_chunk_builder. Kept verbatim as the fallback + / source of truth for the fast path's parity test. - Splits multi-event chunks into individual SSE events - Converts str chunks to generic chunks diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index cbfcd34c4382..235a38b75f9e 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -1,6 +1,6 @@ import asyncio from datetime import datetime -from typing import List, Optional +from typing import List, Optional, Tuple import httpx @@ -45,28 +45,44 @@ async def chunk_processor( litellm_logging_obj=litellm_logging_obj, ) + # Resolve once per stream rather than re-reading the global + + # re-branching on every chunk. ``include_cost_in_streaming_usage`` is + # set at config load and stable for the process, matching how the + # proxy-level streaming fast path resolves it. + cost_injection_active = ( + bool(getattr(litellm, "include_cost_in_streaming_usage", False)) + and bool(model_name) + and endpoint_type in (EndpointType.VERTEX_AI, EndpointType.ANTHROPIC) + ) try: - async for chunk in response.aiter_bytes(): - raw_bytes.append(chunk) - if ( - getattr(litellm, "include_cost_in_streaming_usage", False) - and model_name - ): + if not cost_injection_active: + # Hot path: just buffer for end-of-stream logging and forward. + async for chunk in response.aiter_bytes(): + raw_bytes.append(chunk) + yield chunk + else: + # ``cost_injection_active`` already requires ``model_name`` to + # be truthy; pin to a typed local so mypy narrows ``Optional[str]`` + # -> ``str`` for the per-chunk call site. + assert model_name is not None + resolved_model_name: str = model_name + async for chunk in response.aiter_bytes(): + raw_bytes.append(chunk) if endpoint_type == EndpointType.VERTEX_AI: if "streamRawPredict" in url_route or "rawPredict" in url_route: modified_chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection( - chunk, model_name + chunk, resolved_model_name ) if modified_chunk is not None: chunk = modified_chunk - elif endpoint_type == EndpointType.ANTHROPIC: + else: # EndpointType.ANTHROPIC modified_chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection( - chunk, model_name + chunk, resolved_model_name ) if modified_chunk is not None: chunk = modified_chunk - yield chunk + yield chunk except Exception as e: verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}") raise @@ -115,64 +131,20 @@ async def _route_streaming_logging_to_handler( - OpenAI """ try: - all_chunks = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines( - raw_bytes + ( + standard_logging_response_object, + kwargs, + ) = PassThroughStreamingHandler._build_passthrough_logging_result( + litellm_logging_obj=litellm_logging_obj, + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + request_body=request_body, + endpoint_type=endpoint_type, + start_time=start_time, + raw_bytes=raw_bytes, + end_time=end_time, + model=model, ) - standard_logging_response_object: Optional[ - PassThroughEndpointLoggingResultValues - ] = None - kwargs: dict = {} - if endpoint_type == EndpointType.ANTHROPIC: - anthropic_passthrough_logging_handler_result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( - litellm_logging_obj=litellm_logging_obj, - passthrough_success_handler_obj=passthrough_success_handler_obj, - url_route=url_route, - request_body=request_body, - endpoint_type=endpoint_type, - start_time=start_time, - all_chunks=all_chunks, - end_time=end_time, - ) - standard_logging_response_object = ( - anthropic_passthrough_logging_handler_result["result"] - ) - kwargs = anthropic_passthrough_logging_handler_result["kwargs"] - elif endpoint_type == EndpointType.VERTEX_AI: - vertex_passthrough_logging_handler_result = VertexPassthroughLoggingHandler._handle_logging_vertex_collected_chunks( - litellm_logging_obj=litellm_logging_obj, - passthrough_success_handler_obj=passthrough_success_handler_obj, - url_route=url_route, - request_body=request_body, - endpoint_type=endpoint_type, - start_time=start_time, - all_chunks=all_chunks, - end_time=end_time, - model=model, - ) - standard_logging_response_object = ( - vertex_passthrough_logging_handler_result["result"] - ) - kwargs = vertex_passthrough_logging_handler_result["kwargs"] - elif endpoint_type == EndpointType.OPENAI: - openai_passthrough_logging_handler_result = OpenAIPassthroughLoggingHandler._handle_logging_openai_collected_chunks( - litellm_logging_obj=litellm_logging_obj, - passthrough_success_handler_obj=passthrough_success_handler_obj, - url_route=url_route, - request_body=request_body, - endpoint_type=endpoint_type, - start_time=start_time, - all_chunks=all_chunks, - end_time=end_time, - ) - standard_logging_response_object = ( - openai_passthrough_logging_handler_result["result"] - ) - kwargs = openai_passthrough_logging_handler_result["kwargs"] - - if standard_logging_response_object is None: - standard_logging_response_object = StandardPassThroughResponseObject( - response=f"cannot parse chunks to standard response object. Chunks={all_chunks}" - ) await litellm_logging_obj.async_success_handler( result=standard_logging_response_object, start_time=start_time, @@ -199,6 +171,89 @@ async def _route_streaming_logging_to_handler( f"Error in _route_streaming_logging_to_handler: {str(e)}" ) + @staticmethod + def _build_passthrough_logging_result( + litellm_logging_obj: LiteLLMLoggingObj, + passthrough_success_handler_obj: PassThroughEndpointLogging, + url_route: str, + request_body: dict, + endpoint_type: EndpointType, + start_time: datetime, + raw_bytes: List[bytes], + end_time: datetime, + model: Optional[str], + ) -> Tuple[PassThroughEndpointLoggingResultValues, dict]: + """ + Synchronous, CPU-bound reconstruction of the standard logging payload + from collected raw SSE bytes. Extracted from + _route_streaming_logging_to_handler so the per-endpoint dispatch can + be unit-tested in isolation. Still invoked synchronously on the event + loop; an off-loop dispatch is a future change, not part of this PR. + """ + all_chunks = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines( + raw_bytes + ) + standard_logging_response_object: Optional[ + PassThroughEndpointLoggingResultValues + ] = None + kwargs: dict = {} + if endpoint_type == EndpointType.ANTHROPIC: + anthropic_passthrough_logging_handler_result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=litellm_logging_obj, + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + request_body=request_body, + endpoint_type=endpoint_type, + start_time=start_time, + all_chunks=all_chunks, + end_time=end_time, + ) + standard_logging_response_object = ( + anthropic_passthrough_logging_handler_result["result"] + ) + kwargs = anthropic_passthrough_logging_handler_result["kwargs"] + elif endpoint_type == EndpointType.VERTEX_AI: + vertex_passthrough_logging_handler_result = ( + VertexPassthroughLoggingHandler._handle_logging_vertex_collected_chunks( + litellm_logging_obj=litellm_logging_obj, + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + request_body=request_body, + endpoint_type=endpoint_type, + start_time=start_time, + all_chunks=all_chunks, + end_time=end_time, + model=model, + ) + ) + standard_logging_response_object = ( + vertex_passthrough_logging_handler_result["result"] + ) + kwargs = vertex_passthrough_logging_handler_result["kwargs"] + elif endpoint_type == EndpointType.OPENAI: + openai_passthrough_logging_handler_result = ( + OpenAIPassthroughLoggingHandler._handle_logging_openai_collected_chunks( + litellm_logging_obj=litellm_logging_obj, + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + request_body=request_body, + endpoint_type=endpoint_type, + start_time=start_time, + all_chunks=all_chunks, + end_time=end_time, + ) + ) + standard_logging_response_object = ( + openai_passthrough_logging_handler_result["result"] + ) + kwargs = openai_passthrough_logging_handler_result["kwargs"] + + if standard_logging_response_object is None: + standard_logging_response_object = StandardPassThroughResponseObject( + response=f"cannot parse chunks to standard response object. Chunks={all_chunks}" + ) + return standard_logging_response_object, kwargs + @staticmethod def _extract_model_for_cost_injection( request_body: Optional[dict], diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 032ab6c63b21..14f7f411e415 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1596,7 +1596,22 @@ def _callback_capabilities() -> "_CallbackCapabilities": iterator_overrides.append((resolved, "override")) elif "apply_guardrail" in cls_attrs: iterator_overrides.append((resolved, "apply_guardrail")) - if "async_post_call_streaming_hook" in cls_attrs: + # Walk the MRO for ``async_post_call_streaming_hook`` rather than + # using the leaf-class ``__dict__`` check used by the other flags: + # before this PR the hook was unconditionally invoked, so a + # callback that inherits an override from an intermediate parent + # (e.g. a vendor base class providing the override, with the + # registered class adding nothing else) MUST still be detected. + # A leaf-class miss here would silently drop the inherited hook. + base_streaming_hook = CustomLogger.async_post_call_streaming_hook + cls_streaming_hook = getattr( + cls, + "async_post_call_streaming_hook", + base_streaming_hook, + ) + if getattr( + cls_streaming_hook, "__func__", cls_streaming_hook + ) is not getattr(base_streaming_hook, "__func__", base_streaming_hook): has_streaming_chunk_override = True if "async_pre_call_hook" in cls_attrs: has_pre_call_override = True diff --git a/scripts/benchmark_anthropic_messages_perf.py b/scripts/benchmark_anthropic_messages_perf.py new file mode 100644 index 000000000000..3c8a22f0cc27 --- /dev/null +++ b/scripts/benchmark_anthropic_messages_perf.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Benchmark LiteLLM proxy /v1/messages (Anthropic Messages API) streaming. + +Measures the two metrics that matter for an interactive streaming proxy: + + * TTFT - time to first streamed token (first ``content_block_delta``) + * TPM - sustained output token throughput (tokens / second) once the + full stream is consumed, plus request throughput (RPS) + +It boots a local mock Anthropic provider that speaks the real Anthropic +streaming SSE wire format (``message_start`` -> ``content_block_delta`` -> +``message_stop``) and a LiteLLM proxy from any checkout, so commits/branches +can be compared without depending on real provider latency. + +Example: + uv run python scripts/benchmark_anthropic_messages_perf.py \ + --label baseline --proxy-command ".venv/bin/litellm" + +Compare an already-running proxy: + uv run python scripts/benchmark_anthropic_messages_perf.py \ + --no-start-proxy --label current +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import shlex +import signal +import statistics +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +import aiohttp +from aiohttp import web + +DEFAULT_MODEL = "claude-perf-test" +DEFAULT_API_KEY = "sk-1234" + + +@dataclass +class StreamSample: + success: bool + ttft_ms: float + total_ms: float + output_tokens: int + status_code: int + error: str = "" + + +@dataclass +class SummaryStats: + requests: int + failures: int + rps: float + ttft_mean_ms: float + ttft_p50_ms: float + ttft_p95_ms: float + ttft_p99_ms: float + total_p50_ms: float + total_p95_ms: float + tokens_per_sec: float + + +class MockAnthropicProvider: + """Minimal Anthropic Messages API server (real streaming SSE format).""" + + def __init__( + self, + host: str, + port: int, + first_token_delay_ms: float, + stream_content_chunks: int, + ) -> None: + self.host = host + self.port = port + self.first_token_delay_ms = first_token_delay_ms + self.stream_content_chunks = stream_content_chunks + self.runner: Optional[web.AppRunner] = None + + @property + def base_url(self) -> str: + return f"http://{self.host}:{self.port}" + + async def start(self) -> None: + app = web.Application() + app.router.add_post("/v1/messages", self.handle_messages) + self.runner = web.AppRunner(app, access_log=None) + await self.runner.setup() + site = web.TCPSite(self.runner, self.host, self.port) + await site.start() + + async def stop(self) -> None: + if self.runner is not None: + await self.runner.cleanup() + + async def handle_messages(self, request: web.Request) -> web.StreamResponse: + body = await request.json() + if body.get("stream"): + return await self._streaming_response(request, body) + return self._json_response(body) + + def _json_response(self, body: dict[str, Any]) -> web.Response: + payload = { + "id": "msg_perf", + "type": "message", + "role": "assistant", + "model": body.get("model", DEFAULT_MODEL), + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 8, "output_tokens": 1}, + } + return web.json_response(payload) + + @staticmethod + def _sse(event: str, data: dict[str, Any]) -> bytes: + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + async def _streaming_response( + self, request: web.Request, body: dict[str, Any] + ) -> web.StreamResponse: + model = body.get("model", DEFAULT_MODEL) + response = web.StreamResponse( + status=200, + headers={ + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + }, + ) + await response.prepare(request) + + await response.write( + self._sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_perf", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 8, "output_tokens": 0}, + }, + }, + ) + ) + await response.write( + self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ) + ) + + if self.first_token_delay_ms > 0: + await asyncio.sleep(self.first_token_delay_ms / 1000) + + for _ in range(self.stream_content_chunks): + await response.write( + self._sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "hello "}, + }, + ) + ) + + await response.write( + self._sse("content_block_stop", {"type": "content_block_stop", "index": 0}) + ) + await response.write( + self._sse( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": self.stream_content_chunks}, + }, + ) + ) + await response.write(self._sse("message_stop", {"type": "message_stop"})) + await response.write_eof() + return response + + +def percentile(values: list[float], pct: float) -> float: + if not values: + return 0.0 + sorted_values = sorted(values) + index = min(int(len(sorted_values) * pct / 100), len(sorted_values) - 1) + return sorted_values[index] + + +def summarize(samples: list[StreamSample], wall_time_s: float) -> SummaryStats: + ok = [s for s in samples if s.success] + ttfts = [s.ttft_ms for s in ok] + totals = [s.total_ms for s in ok] + total_tokens = sum(s.output_tokens for s in ok) + return SummaryStats( + requests=len(samples), + failures=len(samples) - len(ok), + rps=(len(ok) / wall_time_s) if wall_time_s > 0 else 0.0, + ttft_mean_ms=statistics.mean(ttfts) if ttfts else 0.0, + ttft_p50_ms=percentile(ttfts, 50), + ttft_p95_ms=percentile(ttfts, 95), + ttft_p99_ms=percentile(ttfts, 99), + total_p50_ms=percentile(totals, 50), + total_p95_ms=percentile(totals, 95), + # Aggregate output-token throughput: total tokens delivered across all + # successful requests divided by wall-clock time. This is the true + # server TPM and (unlike tokens / summed-per-request-latency) scales + # correctly with concurrency. + tokens_per_sec=(total_tokens / wall_time_s) if wall_time_s > 0 else 0.0, + ) + + +def get_git_revision(litellm_dir: Path) -> str: + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + cwd=litellm_dir, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + except Exception: + return "unknown" + + +def write_proxy_config(config_path: Path, provider_base_url: str, api_key: str) -> None: + config_path.write_text( + f"""model_list: + - model_name: {DEFAULT_MODEL} + litellm_params: + model: anthropic/{DEFAULT_MODEL} + api_key: fake-provider-key + api_base: {provider_base_url} + +general_settings: + master_key: {api_key} + +litellm_settings: + telemetry: false +""", + encoding="utf-8", + ) + + +async def wait_for_proxy(base_url: str, timeout_s: float) -> None: + deadline = time.perf_counter() + timeout_s + last_error = "" + async with aiohttp.ClientSession() as session: + while time.perf_counter() < deadline: + try: + async with session.get(f"{base_url}/health/liveliness") as response: + if response.status < 500: + return + last_error = f"HTTP {response.status}" + except Exception as exc: + last_error = str(exc) + await asyncio.sleep(0.5) + raise TimeoutError(f"Timed out waiting for proxy at {base_url}: {last_error}") + + +def start_proxy_process( + litellm_dir: Path, + proxy_command: str, + config_path: Path, + port: int, + log_path: Path, +) -> subprocess.Popen: + command = shlex.split(proxy_command) + [ + "--config", + str(config_path), + "--port", + str(port), + ] + env = { + **os.environ, + "LITELLM_TELEMETRY": "False", + "PYTHONUNBUFFERED": "1", + } + log_file = log_path.open("w", encoding="utf-8") + return subprocess.Popen( + command, + cwd=litellm_dir, + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + +def stop_proxy_process(process: subprocess.Popen) -> None: + if process.poll() is not None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=10) + except Exception: + try: + os.killpg(process.pid, signal.SIGKILL) + except Exception: + pass + + +async def measure_stream( + session: aiohttp.ClientSession, + url: str, + headers: dict[str, str], + payload: dict[str, Any], +) -> StreamSample: + start = time.perf_counter() + ttft_ms = 0.0 + output_tokens = 0 + try: + async with session.post(url, headers=headers, json=payload) as response: + if response.status != 200: + body = await response.read() + return StreamSample( + success=False, + ttft_ms=0.0, + total_ms=(time.perf_counter() - start) * 1000, + output_tokens=0, + status_code=response.status, + error=body.decode("utf-8", errors="ignore")[:200], + ) + async for raw_line in response.content: + line = raw_line.strip() + if not line.startswith(b"data:"): + continue + data = line[5:].strip() + if data == b"[DONE]": + break + try: + event = json.loads(data) + except json.JSONDecodeError: + continue + etype = event.get("type") + if etype == "content_block_delta": + if ttft_ms == 0.0: + ttft_ms = (time.perf_counter() - start) * 1000 + output_tokens += 1 + elif etype == "message_stop": + break + total_ms = (time.perf_counter() - start) * 1000 + if ttft_ms == 0.0: + return StreamSample( + success=False, + ttft_ms=0.0, + total_ms=total_ms, + output_tokens=0, + status_code=response.status, + error="stream ended before a content token", + ) + return StreamSample( + success=True, + ttft_ms=ttft_ms, + total_ms=total_ms, + output_tokens=output_tokens, + status_code=response.status, + ) + except Exception as exc: + return StreamSample( + success=False, + ttft_ms=0.0, + total_ms=(time.perf_counter() - start) * 1000, + output_tokens=0, + status_code=0, + error=str(exc)[:200], + ) + + +async def run_benchmark( + url: str, + headers: dict[str, str], + payload: dict[str, Any], + requests: int, + concurrency: int, + warmup: int, + timeout_s: float, +) -> SummaryStats: + timeout = aiohttp.ClientTimeout(total=timeout_s) + connector = aiohttp.TCPConnector( + limit=max(concurrency * 2, 10), + limit_per_host=max(concurrency, 10), + force_close=False, + ) + + async def worker( + session: aiohttp.ClientSession, + counter: list[int], + budget: int, + sink: list[StreamSample], + ) -> None: + # Steady-state load: exactly `concurrency` workers, each pulling the + # next request slot as soon as its previous one finishes. Keeps + # in-flight concurrency constant (vs. a gather-all + semaphore burst) + # which removes the thundering-herd variance that otherwise swamps a + # 10% signal. + while True: + idx = counter[0] + if idx >= budget: + return + counter[0] = idx + 1 + sink.append(await measure_stream(session, url, headers, payload)) + + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + if warmup > 0: + wcounter = [0] + await asyncio.gather( + *[worker(session, wcounter, warmup, []) for _ in range(concurrency)] + ) + samples: list[StreamSample] = [] + counter = [0] + wall_start = time.perf_counter() + await asyncio.gather( + *[worker(session, counter, requests, samples) for _ in range(concurrency)] + ) + wall_time_s = time.perf_counter() - wall_start + return summarize(samples, wall_time_s) + + +def stats_to_dict(stats: SummaryStats) -> dict[str, Any]: + return { + "requests": stats.requests, + "failures": stats.failures, + "rps": stats.rps, + "ttft_mean_ms": stats.ttft_mean_ms, + "ttft_p50_ms": stats.ttft_p50_ms, + "ttft_p95_ms": stats.ttft_p95_ms, + "ttft_p99_ms": stats.ttft_p99_ms, + "total_p50_ms": stats.total_p50_ms, + "total_p95_ms": stats.total_p95_ms, + "tokens_per_sec": stats.tokens_per_sec, + } + + +def print_summary(label: str, revision: str, stats: SummaryStats) -> None: + print("\n=== Anthropic /v1/messages streaming benchmark ===") + print(f"Label: {label}") + print(f"Revision: {revision}") + print(f"Requests: {stats.requests} Failures: {stats.failures}") + print(f"TTFT mean: {stats.ttft_mean_ms:.2f} ms") + print(f"TTFT p50: {stats.ttft_p50_ms:.2f} ms") + print(f"TTFT p95: {stats.ttft_p95_ms:.2f} ms") + print(f"TTFT p99: {stats.ttft_p99_ms:.2f} ms") + print(f"Full p50: {stats.total_p50_ms:.2f} ms") + print(f"Full p95: {stats.total_p95_ms:.2f} ms") + print(f"Throughput: {stats.rps:.2f} req/s") + print(f"TPM: {stats.tokens_per_sec:.1f} output tokens/s") + print("\nMarkdown row:") + print( + "| " + + " | ".join( + [ + label, + revision, + f"{stats.ttft_p50_ms:.2f}", + f"{stats.ttft_p95_ms:.2f}", + f"{stats.tokens_per_sec:.1f}", + f"{stats.rps:.2f}", + ] + ) + + " |" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--label", default="current") + parser.add_argument("--litellm-dir", default=str(Path.cwd())) + parser.add_argument("--proxy-command", default="uv run litellm") + parser.add_argument("--proxy-host", default="127.0.0.1") + parser.add_argument("--proxy-port", type=int, default=4000) + parser.add_argument("--provider-host", default="127.0.0.1") + parser.add_argument("--provider-port", type=int, default=8098) + parser.add_argument("--api-key", default=DEFAULT_API_KEY) + parser.add_argument("--requests", type=int, default=300) + parser.add_argument("--concurrency", type=int, default=20) + parser.add_argument("--warmup", type=int, default=30) + parser.add_argument("--timeout", type=float, default=30) + parser.add_argument("--proxy-start-timeout", type=float, default=90) + parser.add_argument("--provider-first-token-delay-ms", type=float, default=0) + parser.add_argument( + "--provider-stream-content-chunks", + type=int, + default=64, + help="Number of text delta chunks the mock emits (default 64).", + ) + parser.add_argument( + "--repeats", + type=int, + default=1, + help="Run the suite N times against the same proxy; report the median run.", + ) + parser.add_argument( + "--no-start-proxy", + action="store_true", + help="Benchmark an already-running proxy at --proxy-host/--proxy-port", + ) + parser.add_argument( + "--provider-url", + help="Use an already-running Anthropic-compatible provider", + ) + parser.add_argument("--output-json", help="Write machine-readable results") + return parser.parse_args() + + +async def async_main() -> None: + args = parse_args() + litellm_dir = Path(args.litellm_dir).resolve() + revision = get_git_revision(litellm_dir) + proxy_base_url = f"http://{args.proxy_host}:{args.proxy_port}" + proxy_url = f"{proxy_base_url}/v1/messages" + headers = { + "Authorization": f"Bearer {args.api_key}", + "Content-Type": "application/json", + } + stream_payload = { + "model": DEFAULT_MODEL, + "max_tokens": 256, + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + } + + provider: Optional[MockAnthropicProvider] = None + proxy_process: Optional[subprocess.Popen] = None + with tempfile.TemporaryDirectory(prefix="litellm-anthropic-perf-") as tmp_dir_name: + tmp_dir = Path(tmp_dir_name) + proxy_log_path = tmp_dir / "proxy.log" + if args.provider_url: + provider_base_url = args.provider_url.rstrip("/") + else: + provider = MockAnthropicProvider( + host=args.provider_host, + port=args.provider_port, + first_token_delay_ms=args.provider_first_token_delay_ms, + stream_content_chunks=args.provider_stream_content_chunks, + ) + await provider.start() + provider_base_url = provider.base_url + + config_path = tmp_dir / "config.yaml" + write_proxy_config(config_path, provider_base_url, args.api_key) + + try: + if not args.no_start_proxy: + proxy_process = start_proxy_process( + litellm_dir=litellm_dir, + proxy_command=args.proxy_command, + config_path=config_path, + port=args.proxy_port, + log_path=proxy_log_path, + ) + await wait_for_proxy(proxy_base_url, args.proxy_start_timeout) + + runs: list[SummaryStats] = [] + for run_idx in range(max(1, args.repeats)): + if args.repeats > 1: + print(f"\n--- Run {run_idx + 1}/{args.repeats} ---") + stats = await run_benchmark( + url=proxy_url, + headers=headers, + payload=stream_payload, + requests=args.requests, + concurrency=args.concurrency, + warmup=args.warmup, + timeout_s=args.timeout, + ) + runs.append(stats) + if args.repeats > 1: + print( + f" run {run_idx + 1}: TTFT p50={stats.ttft_p50_ms:.2f}ms " + f"TPM={stats.tokens_per_sec:.1f} tok/s RPS={stats.rps:.2f}" + ) + + stats = sorted(runs, key=lambda s: s.ttft_p50_ms)[len(runs) // 2] + finally: + if proxy_process is not None: + stop_proxy_process(proxy_process) + if provider is not None: + await provider.stop() + + print_summary(args.label, revision, stats) + + if args.output_json: + Path(args.output_json).write_text( + json.dumps( + { + "label": args.label, + "revision": revision, + "proxy_streaming": stats_to_dict(stats), + "proxy_log_path": str(proxy_log_path), + }, + indent=2, + sort_keys=True, + ), + encoding="utf-8", + ) + + +def main() -> None: + asyncio.run(async_main()) + + +if __name__ == "__main__": + main() diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index bd1fe75f3638..b1e1d789d744 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -544,3 +544,132 @@ def test_translate_thinking_for_model_preserves_summary(self): assert result == { "reasoning_effort": {"effort": "medium", "summary": "concise"} } + + +# --------------------------------------------------------------------------- +# Parity tests: redundant empty-text-block sanitization scan removal. +# The async wrapper sanitizes once and tells the handler to skip its second +# (redundant) full-messages scan; the sync entry point still sanitizes. +# --------------------------------------------------------------------------- + + +def _empty_block_msgs(): + return [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": " "}, # whitespace-only -> stripped + {"type": "tool_use", "id": "t", "name": "B", "input": {}}, + ], + } + ] + + +def test_handler_strips_when_no_presanitized_flag(): + """Sync entry point (no async wrapper): handler must still sanitize.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + with patch.object( + handler, + "strip_empty_text_blocks_from_anthropic_messages", + wraps=handler.strip_empty_text_blocks_from_anthropic_messages, + ) as spy: + result = handler.anthropic_messages_handler( + max_tokens=10, + messages=_empty_block_msgs(), + model="anthropic/claude-3-5-sonnet-20241022", + custom_llm_provider="anthropic", + mock_response="hi there", + ) + assert spy.call_count == 1 # sanitized exactly once here + assert result is not None + + +def test_handler_skips_strip_when_presanitized(): + """Async wrapper already sanitized -> handler must NOT rescan.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + with patch.object( + handler, + "strip_empty_text_blocks_from_anthropic_messages", + wraps=handler.strip_empty_text_blocks_from_anthropic_messages, + ) as spy: + result = handler.anthropic_messages_handler( + max_tokens=10, + messages=_empty_block_msgs(), + model="anthropic/claude-3-5-sonnet-20241022", + custom_llm_provider="anthropic", + mock_response="hi there", + _litellm_messages_presanitized=True, + ) + assert spy.call_count == 0 # skipped the redundant scan + assert result is not None + + +def test_presanitized_flag_not_leaked_to_provider_params(): + """The private sentinel must be popped, never forwarded as a request param.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + captured = {} + + def fake_base_handler(*args, **kwargs): + captured.update(kwargs) + captured["optional"] = kwargs.get( + "anthropic_messages_optional_request_params", {} + ) + return "stub" + + with patch.object( + handler.base_llm_http_handler, + "anthropic_messages_handler", + side_effect=fake_base_handler, + ): + handler.anthropic_messages_handler( + max_tokens=10, + messages=[{"role": "user", "content": "hi"}], + model="anthropic/claude-3-5-sonnet-20241022", + custom_llm_provider="anthropic", + _litellm_messages_presanitized=True, + ) + + assert "_litellm_messages_presanitized" not in captured.get("optional", {}) + assert "_litellm_messages_presanitized" not in captured.get("kwargs", {}) + + +@pytest.mark.asyncio +async def test_async_wrapper_sets_presanitized_and_sanitizes_once(): + """End-to-end: wrapper sanitizes (once) AND signals the handler to skip.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + captured = {} + + def fake_handler(*args, **kwargs): + captured["messages"] = kwargs.get("messages") + captured["presanitized"] = kwargs.get("_litellm_messages_presanitized") + return "stub" + + fake_loop = MagicMock() + fake_loop.run_in_executor = lambda _e, func: _async_return(func()) + + with ( + patch.object(handler, "anthropic_messages_handler", side_effect=fake_handler), + patch("asyncio.get_event_loop", return_value=fake_loop), + patch.object( + handler, + "strip_empty_text_blocks_from_anthropic_messages", + wraps=handler.strip_empty_text_blocks_from_anthropic_messages, + ) as spy, + ): + await handler.anthropic_messages( + max_tokens=100, + messages=_empty_block_msgs(), + model="anthropic/claude-sonnet-4-5-20250929", + custom_llm_provider="anthropic", + api_key="k", + ) + + # Wrapper stripped exactly once (the handler is faked, so its skipped + # call never runs anyway -- the point is the wrapper still sanitizes). + assert spy.call_count == 1 + assert captured["presanitized"] is True + assert [b["type"] for b in captured["messages"][0]["content"]] == ["tool_use"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py new file mode 100644 index 000000000000..3ce076640e8b --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py @@ -0,0 +1,56 @@ +""" +Regression tests for the /v1/messages request-parse fast paths: + +- get_requested_anthropic_messages_optional_param must still filter to the + valid AnthropicMessagesRequestOptionalParams keys and drop None values, + while resolving the (static) type hints only once per process. +""" + +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + AnthropicMessagesRequestUtils, + _anthropic_messages_optional_param_keys, +) + + +def test_optional_param_filtering_unchanged(): + params = { + "temperature": 0.5, + "top_p": None, # None dropped + "tools": [{"name": "x"}], + "not_a_real_param": "drop me", # invalid key dropped + "stream": True, + } + result = ( + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params + ) + ) + assert result == {"temperature": 0.5, "tools": [{"name": "x"}], "stream": True} + assert "top_p" not in result + assert "not_a_real_param" not in result + + +def test_valid_keys_are_memoized(): + _anthropic_messages_optional_param_keys.cache_clear() + first = _anthropic_messages_optional_param_keys() + for _ in range(50): + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + {"temperature": 0.1} + ) + info = _anthropic_messages_optional_param_keys.cache_info() + # Resolved exactly once despite many calls. + assert info.misses == 1 + assert info.hits >= 50 + # Stable identity (frozenset) returned each call. + assert _anthropic_messages_optional_param_keys() is first + assert isinstance(first, frozenset) + assert "temperature" in first and "tools" in first + + +def test_empty_params(): + assert ( + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + {} + ) + == {} + ) diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index b846cd600f02..07a61c9c1049 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -101,6 +101,73 @@ def test_get_agentic_loop_settings_defaults_and_overrides(): assert fingerprints == ["fp-1", "fp-2"] +def test_has_agentic_completion_hook_detection(monkeypatch): + """The streaming path skips the agentic wrapper only when no callback + overrides async_should_run_agentic_loop. Verify both directions.""" + from litellm.integrations.custom_logger import CustomLogger + + handler = BaseLLMHTTPHandler() + logging_obj = Mock() + logging_obj.dynamic_success_callbacks = [] + + # No callbacks at all -> no agentic hook. + monkeypatch.setattr(litellm, "callbacks", []) + assert handler._has_agentic_completion_hook(logging_obj) is False + + # A plain CustomLogger that does NOT override the gate -> still no hook + # (so the wrapper is safely skipped). + class _PlainLogger(CustomLogger): + pass + + monkeypatch.setattr(litellm, "callbacks", [_PlainLogger()]) + assert handler._has_agentic_completion_hook(logging_obj) is False + + # A logger that overrides the gate (directly) -> hook present. + class _AgenticLogger(CustomLogger): + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ): + return True, {} + + monkeypatch.setattr(litellm, "callbacks", [_AgenticLogger()]) + assert handler._has_agentic_completion_hook(logging_obj) is True + + # Override inherited through an intermediate class is still detected + # (function-identity check, not a leaf __dict__ check). + class _DerivedAgenticLogger(_AgenticLogger): + pass + + monkeypatch.setattr(litellm, "callbacks", [_DerivedAgenticLogger()]) + assert handler._has_agentic_completion_hook(logging_obj) is True + + # Hook supplied via logging_obj.dynamic_success_callbacks is detected too. + monkeypatch.setattr(litellm, "callbacks", []) + logging_obj.dynamic_success_callbacks = [_AgenticLogger()] + assert handler._has_agentic_completion_hook(logging_obj) is True + + # String-named callback entry (e.g. "datadog") must be resolved to its + # CustomLogger instance via get_custom_logger_compatible_class -- the same + # way ProxyLogging._callback_capabilities handles them. Without that + # resolution a string-registered agentic callback would be silently + # skipped and the buffering wrapper would never fire. + logging_obj.dynamic_success_callbacks = [] + agentic_via_string = _AgenticLogger() + monkeypatch.setattr(litellm, "callbacks", ["fake_string_callback"]) + monkeypatch.setattr( + "litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class", + lambda name: agentic_via_string if name == "fake_string_callback" else None, + ) + assert handler._has_agentic_completion_hook(logging_obj) is True + + # Unresolvable string (returns None) is skipped, no false positive. + monkeypatch.setattr(litellm, "callbacks", ["unknown_callback"]) + monkeypatch.setattr( + "litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class", + lambda name: None, + ) + assert handler._has_agentic_completion_hook(logging_obj) is False + + def test_fingerprint_agentic_tools_is_deterministic(): handler = BaseLLMHTTPHandler() tools_a = {"tool_calls": [{"id": "1", "input": {"q": "abc"}, "name": "web_search"}]} @@ -422,3 +489,143 @@ def test_sync_delete_responses_omits_body_for_azure(): assert captured["url"].endswith( "/openai/responses/resp_xyz?api-version=2025-03-01-preview" ) + + +# --------------------------------------------------------------------------- +# Parity tests: request-body is serialized once and reused for the wire. +# (_async_post_anthropic_messages_with_http_error_retry) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_anthropic_post_uses_prebuilt_body_without_redumping(): + """When the caller passes a pre-serialized (unsigned) body, attempt 0 must + send exactly those bytes -- no second json.dumps of request_body.""" + import json as _json + + handler = BaseLLMHTTPHandler() + request_body = {"model": "claude", "messages": [{"role": "user", "content": "hi"}]} + prebuilt = _json.dumps(request_body) + + ok_resp = Mock() + ok_resp.raise_for_status = Mock(return_value=None) + http_client = Mock() + http_client.post = AsyncMock(return_value=ok_resp) + + provider_config = Mock() + provider_config.max_retry_on_anthropic_messages_http_error = 2 + + logging_obj = Mock() + logging_obj.model_call_details = {} + + out = await handler._async_post_anthropic_messages_with_http_error_retry( + async_httpx_client=http_client, + request_url="http://x/v1/messages", + headers={}, + signed_json_body=prebuilt, + request_body=request_body, + stream=False, + logging_obj=logging_obj, + provider_config=provider_config, + litellm_params=GenericLiteLLMParams(), + api_key="k", + model="claude", + ) + assert out is ok_resp + http_client.post.assert_awaited_once() + sent = http_client.post.await_args.kwargs["data"] + # Byte-identical to the legacy wire serialization, and the SAME object the + # caller already used for the pre-call log (no re-serialization). + assert sent == prebuilt + assert sent is prebuilt + + +@pytest.mark.asyncio +async def test_anthropic_post_falls_back_to_json_dumps_when_unsigned_none(): + """signed_json_body=None keeps the exact legacy behavior.""" + import json as _json + + handler = BaseLLMHTTPHandler() + request_body = {"model": "claude", "messages": [{"role": "user", "content": "yo"}]} + + ok_resp = Mock() + ok_resp.raise_for_status = Mock(return_value=None) + http_client = Mock() + http_client.post = AsyncMock(return_value=ok_resp) + + provider_config = Mock() + provider_config.max_retry_on_anthropic_messages_http_error = 1 + logging_obj = Mock() + logging_obj.model_call_details = {} + + await handler._async_post_anthropic_messages_with_http_error_retry( + async_httpx_client=http_client, + request_url="http://x/v1/messages", + headers={}, + signed_json_body=None, + request_body=request_body, + stream=False, + logging_obj=logging_obj, + provider_config=provider_config, + litellm_params=GenericLiteLLMParams(), + api_key="k", + model="claude", + ) + sent = http_client.post.await_args.kwargs["data"] + assert sent == _json.dumps(request_body) + + +@pytest.mark.asyncio +async def test_anthropic_post_retry_reserializes_mutated_body(): + """On a retryable HTTP error the body is mutated + re-signed; the prebuilt + body must NOT be reused -- attempt 1 sends the freshly serialized body.""" + import json as _json + + handler = BaseLLMHTTPHandler() + request_body = {"model": "claude", "messages": [{"role": "user", "content": "a"}]} + prebuilt = _json.dumps(request_body) + + err_resp = Mock() + http_error = httpx.HTTPStatusError( + "bad", request=Mock(), response=Mock(status_code=400) + ) + err_resp.raise_for_status = Mock(side_effect=http_error) + ok_resp = Mock() + ok_resp.raise_for_status = Mock(return_value=None) + http_client = Mock() + http_client.post = AsyncMock(side_effect=[err_resp, ok_resp]) + + def _mutate(e, request_data): + request_data["messages"][0]["content"] = "MUTATED" + + provider_config = Mock() + provider_config.max_retry_on_anthropic_messages_http_error = 2 + provider_config.should_retry_anthropic_messages_on_http_error = Mock( + return_value=True + ) + provider_config.transform_anthropic_messages_request_on_http_error = _mutate + # Re-sign returns no signed body (native anthropic path) -> must re-dump. + provider_config.sign_request = Mock(return_value=({}, None)) + + logging_obj = Mock() + logging_obj.model_call_details = {} + + await handler._async_post_anthropic_messages_with_http_error_retry( + async_httpx_client=http_client, + request_url="http://x/v1/messages", + headers={}, + signed_json_body=prebuilt, + request_body=request_body, + stream=False, + logging_obj=logging_obj, + provider_config=provider_config, + litellm_params=GenericLiteLLMParams(), + api_key="k", + model="claude", + ) + assert http_client.post.await_count == 2 + first_sent = http_client.post.await_args_list[0].kwargs["data"] + second_sent = http_client.post.await_args_list[1].kwargs["data"] + assert first_sent == prebuilt # attempt 0 used prebuilt + assert second_sent == _json.dumps(request_body) # attempt 1 re-serialized + assert "MUTATED" in second_sent # ... the mutated body diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index c16c42decc09..0a9e30310303 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -684,3 +684,362 @@ def test_store_batch_managed_object_success( mock_proxy_logging_obj.get_proxy_hook.assert_called_once_with( "managed_files" ) + + +class TestPureTextFastPathParity: + """ + The pure-text fast path in _build_complete_streaming_response must produce + a response (and downstream logging/cost payload) byte-identical to the + legacy stream_chunk_builder path. Anything non-text must fall back. + """ + + @staticmethod + def _sse(event, data): + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + @staticmethod + def _to_all_chunks(raw_frames): + # Mirror production: raw bytes -> _convert_raw_bytes_to_str_lines. + from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, + ) + + return PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_frames) + + @staticmethod + def _norm(resp): + if resp is None: + return None + d = resp.model_dump() + # id / created are non-deterministic even between two legacy runs. + d.pop("id", None) + d.pop("created", None) + return d + + def _text_stream( + self, + texts, + *, + input_tokens=12, + cache_creation=0, + cache_read=0, + stop_reason="end_turn", + with_ping=True, + blocks=1, + ): + frames = [ + self._sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_abc", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20241022", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": input_tokens, + "output_tokens": 0, + "cache_creation_input_tokens": cache_creation, + "cache_read_input_tokens": cache_read, + }, + }, + }, + ) + ] + per_block = max(1, len(texts) // blocks) + ti = 0 + for b in range(blocks): + frames.append( + self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": b, + "content_block": {"type": "text", "text": ""}, + }, + ) + ) + if with_ping: + frames.append(self._sse("ping", {"type": "ping"})) + chunk_texts = texts[ti : ti + per_block] if b < blocks - 1 else texts[ti:] + ti += per_block + for t in chunk_texts: + frames.append( + self._sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": b, + "delta": {"type": "text_delta", "text": t}, + }, + ) + ) + frames.append( + self._sse( + "content_block_stop", {"type": "content_block_stop", "index": b} + ) + ) + frames.append( + self._sse( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": {"output_tokens": len(texts)}, + }, + ) + ) + frames.append(self._sse("message_stop", {"type": "message_stop"})) + return frames + + def _assert_parity(self, raw_frames): + all_chunks = self._to_all_chunks(raw_frames) + lo1 = MagicMock() + lo1.model_call_details = {} + lo2 = MagicMock() + lo2.model_call_details = {} + + legacy = AnthropicPassthroughLoggingHandler._build_complete_streaming_response_legacy( + all_chunks=list(all_chunks), + litellm_logging_obj=lo1, + model="claude-3-5-sonnet-20241022", + ) + fast = AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=list(all_chunks), + litellm_logging_obj=lo2, + model="claude-3-5-sonnet-20241022", + ) + assert self._norm(fast) == self._norm(legacy) + + # Downstream logged/billed payload must also match. + start = datetime.now() + end = datetime.now() + k_legacy = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=legacy, + model="claude-3-5-sonnet-20241022", + kwargs={}, + start_time=start, + end_time=end, + logging_obj=lo1, + ) + k_fast = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=fast, + model="claude-3-5-sonnet-20241022", + kwargs={}, + start_time=start, + end_time=end, + logging_obj=lo2, + ) + # Usage drives cost; it must be byte-identical between paths. + assert getattr(fast, "usage", None) == getattr(legacy, "usage", None) + + # And the full logged payload (sans non-deterministic response id). + def _scrub(p): + d = dict(p) + r = d.get("complete_streaming_response_in_db") or d.get( + "complete_streaming_response" + ) + return d, getattr(r, "usage", None) + + assert _scrub(k_fast)[1] == _scrub(k_legacy)[1] + + def test_parity_simple_text(self): + self._assert_parity(self._text_stream(["Hello", " ", "world", "!"])) + + def test_parity_single_delta(self): + self._assert_parity(self._text_stream(["Just one piece of text."])) + + def test_parity_cache_tokens(self): + self._assert_parity( + self._text_stream( + ["a", "b", "c"], input_tokens=20, cache_creation=5, cache_read=7 + ) + ) + + def test_parity_max_tokens_stop(self): + self._assert_parity(self._text_stream(["tok"] * 8, stop_reason="max_tokens")) + + def test_parity_no_ping(self): + self._assert_parity(self._text_stream(["x", "y"], with_ping=False)) + + def test_parity_empty_text_deltas(self): + self._assert_parity(self._text_stream(["", "hi", "", "there"])) + + def test_parity_multi_text_block(self): + self._assert_parity(self._text_stream(["p1", "p2", "p3", "p4"], blocks=2)) + + def test_parity_multibyte_batched_frames(self): + # Several SSE events delivered in one network chunk. + frames = self._text_stream(["alpha", "beta", "gamma"]) + merged = b"".join(frames) + self._assert_parity([merged]) + + def test_collapse_returns_none_for_tool_use(self): + frames = [ + self._sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "m", + "model": "x", + "role": "assistant", + "type": "message", + "content": [], + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + ), + self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "t1", + "name": "get_weather", + "input": {}, + }, + }, + ), + self._sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": "{}"}, + }, + ), + self._sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + self._sse("message_stop", {"type": "message_stop"}), + ] + all_chunks = self._to_all_chunks(frames) + assert ( + AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks( + list(all_chunks) + ) + is None + ) + + def test_collapse_returns_none_for_thinking(self): + frames = [ + self._sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "m", + "model": "x", + "role": "assistant", + "type": "message", + "content": [], + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + ), + self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": ""}, + }, + ), + self._sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "hmm"}, + }, + ), + self._sse("message_stop", {"type": "message_stop"}), + ] + all_chunks = self._to_all_chunks(frames) + assert ( + AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks( + list(all_chunks) + ) + is None + ) + + def test_collapse_actually_shrinks_chunk_count(self): + frames = self._text_stream(["a"] * 50) + all_chunks = list(self._to_all_chunks(frames)) + collapsed = AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks( + all_chunks + ) + assert collapsed is not None + # 50 text deltas + 50 event markers + 1 ping collapse to far fewer. + assert len(collapsed) < len(all_chunks) / 2 + + def test_collapse_returns_none_for_interleaved_block_indexes(self): + """ + Anthropic sends content blocks strictly sequentially (start/deltas/stop + for one, then the next). If a stream ever interleaves deltas across + block indexes, the fast path must bail to legacy rather than merge text + from different blocks under a single index. + """ + frames = [ + self._sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_abc", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20241022", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + ), + self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "text", "text": ""}, + }, + ), + # Interleave: delta for block 0, then delta for block 1, with no + # content_block_stop between them. + self._sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "hello "}, + }, + ), + self._sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "text_delta", "text": "world"}, + }, + ), + self._sse("message_stop", {"type": "message_stop"}), + ] + all_chunks = list(self._to_all_chunks(frames)) + assert ( + AnthropicPassthroughLoggingHandler._collapse_pure_text_chunks(all_chunks) + is None + ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 31a25916e2e0..265a82d4a441 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -10,6 +10,7 @@ import litellm from litellm._uuid import uuid +from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -1316,8 +1317,17 @@ async def mock_generator(): yield 'data: {"content": "chunk 3"}\n\n' yield "data: [DONE]\n\n" - # Patch the tracer in the common_request_processing module - with patch("litellm.proxy.common_request_processing.tracer", mock_tracer): + # Patch the tracer in the common_request_processing module. The + # per-chunk span is gated on _DD_STREAMING_TRACE_ENABLED (resolved at + # import from the real tracer, a NullTracer by default), so enable it + # explicitly to exercise the tracing path. + with ( + patch("litellm.proxy.common_request_processing.tracer", mock_tracer), + patch( + "litellm.proxy.common_request_processing._DD_STREAMING_TRACE_ENABLED", + True, + ), + ): response = await create_response(mock_generator(), "text/event-stream", {}) assert response.status_code == 200 @@ -1345,6 +1355,40 @@ async def mock_generator(): args[0] == "streaming.chunk.yield" ), f"Call {i} should have operation name 'streaming.chunk.yield', got {args[0]}" + async def test_create_streaming_response_skips_dd_trace_when_disabled(self): + """When DD tracing is disabled (the default), the per-chunk span + context manager is skipped entirely but all chunks still stream.""" + from unittest.mock import patch + + mock_tracer = MagicMock() + + async def mock_generator(): + yield 'data: {"content": "chunk 1"}\n\n' + yield 'data: {"content": "chunk 2"}\n\n' + yield "data: [DONE]\n\n" + + with ( + patch("litellm.proxy.common_request_processing.tracer", mock_tracer), + patch( + "litellm.proxy.common_request_processing._DD_STREAMING_TRACE_ENABLED", + False, + ), + ): + response = await create_response(mock_generator(), "text/event-stream", {}) + + assert response.status_code == 200 + + content = await self.consume_stream(response) + + # All chunks stream through unchanged ... + assert content == [ + 'data: {"content": "chunk 1"}\n\n', + 'data: {"content": "chunk 2"}\n\n', + "data: [DONE]\n\n", + ] + # ... but no per-chunk span was created. + assert mock_tracer.trace.call_count == 0 + async def test_create_streaming_response_dd_trace_with_error_chunk(self): """ Test that when the first chunk contains an error, JSONResponse is returned @@ -2199,3 +2243,77 @@ async def test_string_detail_unchanged(self): proxy_exc = await self._invoke(exc) assert proxy_exc.message == "Content blocked by guardrail" assert proxy_exc.provider_specific_fields is None + + +class TestAsyncStreamingDataGeneratorFastPath: + """Fast/slow path branching in async_streaming_data_generator.""" + + @staticmethod + async def _aiter(items): + for item in items: + yield item + + @pytest.mark.asyncio + async def test_fast_path_skips_per_chunk_hook(self, monkeypatch): + """With no callbacks/guardrails/cost-injection, chunks pass through + unchanged and the per-chunk hook is NOT awaited.""" + monkeypatch.setattr(litellm, "callbacks", []) + ProxyLogging._callback_capabilities_cache.clear() + + proxy_logging_obj = ProxyLogging(user_api_key_cache=MagicMock()) + hook_spy = AsyncMock(side_effect=lambda **kw: kw["response"]) + monkeypatch.setattr( + proxy_logging_obj, "async_post_call_streaming_hook", hook_spy + ) + + chunks = [b"event: a\ndata: {}\n\n", b"event: b\ndata: {}\n\n"] + out = [ + c + async for c in ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=self._aiter(chunks), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + request_data={"model": "claude-x"}, + proxy_logging_obj=proxy_logging_obj, + serialize_chunk=ProxyBaseLLMRequestProcessing.return_sse_chunk, + serialize_error=lambda e: "data: error\n\n", + ) + ] + + assert out == chunks # bytes pass through return_sse_chunk untouched + hook_spy.assert_not_awaited() + + @pytest.mark.asyncio + async def test_slow_path_runs_per_chunk_hook(self, monkeypatch): + """A callback that overrides async_post_call_streaming_hook forces the + slow path and the per-chunk hook is invoked.""" + + class _StreamingCb(CustomLogger): + async def async_post_call_streaming_hook(self, user_api_key_dict, response): + return response + + cb = _StreamingCb() + monkeypatch.setattr(litellm, "callbacks", [cb]) + ProxyLogging._callback_capabilities_cache.clear() + + proxy_logging_obj = ProxyLogging(user_api_key_cache=MagicMock()) + hook_spy = AsyncMock(side_effect=lambda **kw: kw["response"]) + monkeypatch.setattr( + proxy_logging_obj, "async_post_call_streaming_hook", hook_spy + ) + + out = [ + c + async for c in ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=self._aiter([{"type": "message_stop"}]), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + request_data={"model": "claude-x"}, + proxy_logging_obj=proxy_logging_obj, + serialize_chunk=ProxyBaseLLMRequestProcessing.return_sse_chunk, + serialize_error=lambda e: "data: error\n\n", + ) + ] + + assert len(out) == 1 + hook_spy.assert_awaited_once() + + ProxyLogging._callback_capabilities_cache.clear() diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index 4aebcf40aa5f..f5967030561a 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -111,6 +111,28 @@ async def async_post_call_streaming_iterator_hook( # type: ignore[override] assert kind == "override" +def test_callback_capabilities_detects_inherited_streaming_chunk_override(monkeypatch): + """ + ``async_post_call_streaming_hook`` must be detected even when the override + lives on an intermediate parent class — a vendor base class can carry the + override and the registered class can add nothing else. Before this PR the + hook was unconditionally invoked, so a leaf-class ``__dict__`` miss here + would silently drop the inherited hook. + """ + ProxyLogging._callback_capabilities_cache.clear() + + class _StreamingBase(CustomLogger): + async def async_post_call_streaming_hook(self, *args, **kwargs): # type: ignore[override] + return kwargs.get("response") + + class _LeafWithoutOverride(_StreamingBase): + pass + + monkeypatch.setattr(litellm, "callbacks", [_LeafWithoutOverride()]) + caps = ProxyLogging._callback_capabilities() + assert caps.has_streaming_chunk_override is True + + def test_callback_capabilities_cache_invalidates_on_list_change(monkeypatch): """The cache key includes (length, id-of-each-callback). Mutating the callback list must produce a fresh capability snapshot.""" From 203b529c9d74f586bbedbaf7978d4d55da074655 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Sat, 23 May 2026 12:16:01 -0700 Subject: [PATCH 004/109] feat(azure): add speech transcription config support (#27482) Co-authored-by: oss-agent-shin <279349115+oss-agent-shin@users.noreply.github.com> Co-authored-by: ishaan-berri --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 5 + .../azure/audio_transcription/__init__.py | 3 + .../audio_transcription/transformation.py | 224 ++++++++++++++++ litellm/llms/custom_httpx/llm_http_handler.py | 2 + litellm/main.py | 5 +- ...odel_prices_and_context_window_backup.json | 11 + litellm/utils.py | 42 +++ model_prices_and_context_window.json | 11 + .../test_azure_speech_audio_transcription.py | 239 ++++++++++++++++++ 10 files changed, 542 insertions(+), 3 deletions(-) create mode 100644 litellm/llms/azure/audio_transcription/__init__.py create mode 100644 litellm/llms/azure/audio_transcription/transformation.py create mode 100644 tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 3365abe32565..7c92623358d2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1877,6 +1877,9 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: from .llms.azure.completion.transformation import ( AzureOpenAITextConfig as AzureOpenAITextConfig, ) + from .llms.azure.audio_transcription.transformation import ( + AzureSpeechAudioTranscriptionConfig as AzureSpeechAudioTranscriptionConfig, + ) from .llms.hosted_vllm.chat.transformation import ( HostedVLLMChatConfig as HostedVLLMChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index e3656b340fa9..17eb66092924 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -273,6 +273,7 @@ "AzureOpenAIConfig", "AzureOpenAIGPT5Config", "AzureOpenAITextConfig", + "AzureSpeechAudioTranscriptionConfig", "HostedVLLMChatConfig", "HostedVLLMEmbeddingConfig", # Alias for backwards compatibility @@ -1054,6 +1055,10 @@ ".llms.azure.completion.transformation", "AzureOpenAITextConfig", ), + "AzureSpeechAudioTranscriptionConfig": ( + ".llms.azure.audio_transcription.transformation", + "AzureSpeechAudioTranscriptionConfig", + ), "HostedVLLMChatConfig": ( ".llms.hosted_vllm.chat.transformation", "HostedVLLMChatConfig", diff --git a/litellm/llms/azure/audio_transcription/__init__.py b/litellm/llms/azure/audio_transcription/__init__.py new file mode 100644 index 000000000000..cedd0c6dbeb0 --- /dev/null +++ b/litellm/llms/azure/audio_transcription/__init__.py @@ -0,0 +1,3 @@ +from .transformation import AzureSpeechAudioTranscriptionConfig + +__all__ = ["AzureSpeechAudioTranscriptionConfig"] diff --git a/litellm/llms/azure/audio_transcription/transformation.py b/litellm/llms/azure/audio_transcription/transformation.py new file mode 100644 index 000000000000..e478c8ebf350 --- /dev/null +++ b/litellm/llms/azure/audio_transcription/transformation.py @@ -0,0 +1,224 @@ +""" +Azure AI Speech (Cognitive Services) speech-to-text transformation. + +Maps OpenAI-compatible audio transcription calls to Azure Speech REST +recognition for short audio. +""" + +from typing import Any, Dict, List, Optional, Union +from urllib.parse import urlencode, urlparse + +import httpx + +from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIAudioTranscriptionOptionalParams, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import FileTypes, TranscriptionResponse + + +class AzureSpeechAudioTranscriptionException(BaseLLMException): + pass + + +class AzureSpeechAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + """ + Configuration for Azure AI Speech (Cognitive Services) STT. + + Reference: + https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-speech-to-text-short + """ + + COGNITIVE_SERVICES_DOMAIN = "api.cognitive.microsoft.com" + STT_SPEECH_DOMAIN = "stt.speech.microsoft.com" + STT_ENDPOINT_PATH = "/speech/recognition/conversation/cognitiveservices/v1" + DEFAULT_LANGUAGE = "en-US" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIAudioTranscriptionOptionalParams]: + return ["language", "response_format"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model=model) + for key, value in non_default_params.items(): + if key in supported_params: + optional_params[key] = value + return optional_params + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + api_key = api_key or get_secret_str("AZURE_SPEECH_API_KEY") + if not api_key: + raise AzureSpeechAudioTranscriptionException( + message="api_key is required for Azure AI Speech transcription.", + status_code=401, + ) + + validated_headers = headers.copy() + validated_headers["Ocp-Apim-Subscription-Key"] = api_key + validated_headers["Content-Type"] = validated_headers.get( + "Content-Type", "audio/wav" + ) + validated_headers["Accept"] = "application/json" + return validated_headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = api_base or get_secret_str("AZURE_SPEECH_API_BASE") + if api_base is None: + raise AzureSpeechAudioTranscriptionException( + message=( + "api_base is required for Azure AI Speech transcription. " + "Use a Cognitive Services endpoint like " + "https://{region}.api.cognitive.microsoft.com or an STT " + "endpoint like https://{region}.stt.speech.microsoft.com." + ), + status_code=400, + ) + + base_url = self._resolve_stt_base_url(api_base=api_base) + query_params = { + "language": optional_params.get("language", self.DEFAULT_LANGUAGE), + "format": self._get_azure_response_format( + optional_params.get("response_format") + ), + } + return f"{base_url}{self.STT_ENDPOINT_PATH}?{urlencode(query_params)}" + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + processed_audio = process_audio_file(audio_file) + return AudioTranscriptionRequestData( + data=processed_audio.file_content, + files=None, + content_type=processed_audio.content_type, + ) + + def transform_audio_transcription_response( + self, + raw_response: httpx.Response, + ) -> TranscriptionResponse: + response_json = raw_response.json() + recognition_status = response_json.get("RecognitionStatus") + if recognition_status is not None and recognition_status != "Success": + raise AzureSpeechAudioTranscriptionException( + message=( + "Azure AI Speech transcription failed with " + f"RecognitionStatus={recognition_status}." + ), + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + text = self._extract_text(response_json) + response = TranscriptionResponse(text=text) + response._hidden_params = response_json + return response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return AzureSpeechAudioTranscriptionException( + message=error_message, + status_code=status_code, + headers=headers, + ) + + def _resolve_stt_base_url(self, api_base: str) -> str: + api_base = api_base.rstrip("/") + parsed_url = urlparse(api_base) + hostname = parsed_url.hostname or "" + + if self._is_cognitive_services_endpoint(hostname=hostname): + region = self._extract_region_from_hostname( + hostname=hostname, domain=self.COGNITIVE_SERVICES_DOMAIN + ) + return self._build_stt_base_url(region=region) + + if self._is_stt_endpoint(hostname=hostname): + return f"{parsed_url.scheme}://{hostname}" + + if self._is_azure_openai_endpoint(hostname=hostname): + raise AzureSpeechAudioTranscriptionException( + message=( + "Azure AI Speech transcription requires a Cognitive Services " + "or STT Speech endpoint, not an Azure OpenAI endpoint." + ), + status_code=400, + ) + + return api_base + + def _is_cognitive_services_endpoint(self, hostname: str) -> bool: + return hostname == self.COGNITIVE_SERVICES_DOMAIN or hostname.endswith( + f".{self.COGNITIVE_SERVICES_DOMAIN}" + ) + + def _is_stt_endpoint(self, hostname: str) -> bool: + return hostname == self.STT_SPEECH_DOMAIN or hostname.endswith( + f".{self.STT_SPEECH_DOMAIN}" + ) + + def _is_azure_openai_endpoint(self, hostname: str) -> bool: + return hostname.endswith(".openai.azure.com") + + def _extract_region_from_hostname(self, hostname: str, domain: str) -> str: + if hostname.endswith(f".{domain}"): + return hostname[: -len(f".{domain}")] + return "" + + def _build_stt_base_url(self, region: str) -> str: + if region: + return f"https://{region}.{self.STT_SPEECH_DOMAIN}" + return f"https://{self.STT_SPEECH_DOMAIN}" + + def _get_azure_response_format(self, response_format: Optional[str]) -> str: + if response_format == "verbose_json": + return "detailed" + return "simple" + + def _extract_text(self, response_json: Dict[str, Any]) -> str: + if isinstance(response_json.get("DisplayText"), str): + return response_json["DisplayText"] + + nbest = response_json.get("NBest") + if isinstance(nbest, list) and nbest: + best = nbest[0] + if isinstance(best, dict): + return best.get("Display") or best.get("Lexical") or "" + + return "" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 74e55b9debb9..c9ab3c648acd 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1207,6 +1207,8 @@ def _prepare_audio_transcription_request( data = transformed_result.data files = transformed_result.files + if transformed_result.content_type is not None: + headers["Content-Type"] = transformed_result.content_type ## LOGGING logging_obj.pre_call( diff --git a/litellm/main.py b/litellm/main.py index 510e6e424a2e..09c70998cf7d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6615,8 +6615,7 @@ def transcription( api_key=api_key, ) # type: ignore - if dynamic_api_key is not None: - api_key = dynamic_api_key + api_key = dynamic_api_key if dynamic_api_key is not None else api_key optional_params = get_optional_params_transcription( model=model, @@ -6656,7 +6655,7 @@ def transcription( provider=LlmProviders(custom_llm_provider), ) - if custom_llm_provider == "azure": + if custom_llm_provider == "azure" and provider_config is None: # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 08c06b17ddf8..62e576ea0f7f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6054,6 +6054,17 @@ "mode": "audio_speech", "source": "https://azure.microsoft.com/en-us/pricing/calculator/" }, + "azure/speech/azure-stt": { + "audio_transcription_config": "azure_speech", + "input_cost_per_second": 0.0002777778, + "litellm_provider": "azure", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/speech-services/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, "azure/tts-1": { "input_cost_per_character": 1.5e-05, "litellm_provider": "azure", diff --git a/litellm/utils.py b/litellm/utils.py index 2a94941f6563..2ba6ef9cae87 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8146,6 +8146,35 @@ def validate_openai_optional_params( return stop +@lru_cache(maxsize=1) +def _get_bundled_model_cost_map() -> Dict[str, Any]: + try: + model_cost_path = resources.files("litellm").joinpath( + "model_prices_and_context_window_backup.json" + ) + return json.loads(model_cost_path.read_text()) + except Exception: + return {} + + +def _get_model_cost_entry_for_provider_config( + model: str, + provider: LlmProviders, +) -> Dict[str, Any]: + candidate_keys = (model, f"{provider.value}/{model}") + for model_key in candidate_keys: + model_info = litellm.model_cost.get(model_key) + if model_info is not None: + return model_info + + bundled_model_cost = _get_bundled_model_cost_map() + for model_key in candidate_keys: + model_info = bundled_model_cost.get(model_key) + if model_info is not None: + return model_info + return {} + + class ProviderConfigManager: # Dictionary mapping for O(1) provider lookup # Stores tuples of (factory_function, needs_model_parameter) @@ -8621,6 +8650,19 @@ def get_provider_audio_transcription_config( model: str, provider: LlmProviders, ) -> Optional[BaseAudioTranscriptionConfig]: + model_cost_entry = _get_model_cost_entry_for_provider_config( + model=model, + provider=provider, + ) + if ( + litellm.LlmProviders.AZURE == provider + and model_cost_entry.get("audio_transcription_config") == "azure_speech" + ): + from litellm.llms.azure.audio_transcription.transformation import ( + AzureSpeechAudioTranscriptionConfig, + ) + + return AzureSpeechAudioTranscriptionConfig() if litellm.LlmProviders.FIREWORKS_AI == provider: return litellm.FireworksAIAudioTranscriptionConfig() elif litellm.LlmProviders.DEEPGRAM == provider: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 514862516e82..62553e46ac1a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6067,6 +6067,17 @@ "mode": "audio_speech", "source": "https://azure.microsoft.com/en-us/pricing/calculator/" }, + "azure/speech/azure-stt": { + "audio_transcription_config": "azure_speech", + "input_cost_per_second": 0.0002777778, + "litellm_provider": "azure", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/speech-services/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, "azure/tts-1": { "input_cost_per_character": 1.5e-05, "litellm_provider": "azure", diff --git a/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py b/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py new file mode 100644 index 000000000000..6ed6be6f34fa --- /dev/null +++ b/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py @@ -0,0 +1,239 @@ +import io +import json +from pathlib import Path +from unittest.mock import MagicMock + +import httpx +import pytest + +import litellm +from litellm.llms.azure.audio_transcription.transformation import ( + AzureSpeechAudioTranscriptionConfig, + AzureSpeechAudioTranscriptionException, +) +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from litellm.types.utils import TranscriptionResponse +from litellm.utils import ProviderConfigManager + + +def test_azure_speech_audio_transcription_config_installed(): + config = ProviderConfigManager.get_provider_audio_transcription_config( + model="speech/azure-stt", + provider=litellm.LlmProviders.AZURE, + ) + + assert isinstance(config, BaseAudioTranscriptionConfig) + assert isinstance(config, AzureSpeechAudioTranscriptionConfig) + + +def test_azure_speech_audio_transcription_builds_stt_url_from_cognitive_endpoint(): + config = AzureSpeechAudioTranscriptionConfig() + + url = config.get_complete_url( + api_base="https://eastus.api.cognitive.microsoft.com/", + api_key="test-key", + model="speech/azure-stt", + optional_params={"language": "fr-FR", "response_format": "verbose_json"}, + litellm_params={}, + ) + + assert ( + url + == "https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=fr-FR&format=detailed" + ) + + +def test_azure_speech_audio_transcription_accepts_stt_endpoint_base(): + config = AzureSpeechAudioTranscriptionConfig() + + url = config.get_complete_url( + api_base="https://westus.stt.speech.microsoft.com", + api_key="test-key", + model="speech/azure-stt", + optional_params={}, + litellm_params={}, + ) + + assert ( + url + == "https://westus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US&format=simple" + ) + + +def test_azure_speech_audio_transcription_uses_dedicated_api_base_env(monkeypatch): + config = AzureSpeechAudioTranscriptionConfig() + + monkeypatch.setattr( + "litellm.llms.azure.audio_transcription.transformation.get_secret_str", + lambda key: ( + "https://centralus.api.cognitive.microsoft.com" + if key == "AZURE_SPEECH_API_BASE" + else None + ), + ) + + url = config.get_complete_url( + api_base=None, + api_key="test-key", + model="speech/azure-stt", + optional_params={}, + litellm_params={}, + ) + + assert ( + url + == "https://centralus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US&format=simple" + ) + + +def test_azure_speech_audio_transcription_rejects_azure_openai_endpoint(): + config = AzureSpeechAudioTranscriptionConfig() + + with pytest.raises( + AzureSpeechAudioTranscriptionException, + match="not an Azure OpenAI endpoint", + ): + config.get_complete_url( + api_base="https://example.openai.azure.com", + api_key="test-key", + model="speech/azure-stt", + optional_params={}, + litellm_params={}, + ) + + +def test_azure_speech_audio_transcription_validate_environment(): + config = AzureSpeechAudioTranscriptionConfig() + + headers = config.validate_environment( + headers={}, + model="speech/azure-stt", + messages=[], + optional_params={}, + litellm_params={}, + api_key="test-key", + ) + + assert headers["Ocp-Apim-Subscription-Key"] == "test-key" + assert headers["Content-Type"] == "audio/wav" + assert headers["Accept"] == "application/json" + + +def test_azure_speech_audio_transcription_uses_dedicated_api_key_env(monkeypatch): + config = AzureSpeechAudioTranscriptionConfig() + + monkeypatch.setattr( + "litellm.llms.azure.audio_transcription.transformation.get_secret_str", + lambda key: "speech-key" if key == "AZURE_SPEECH_API_KEY" else None, + ) + + headers = config.validate_environment( + headers={}, + model="speech/azure-stt", + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + assert headers["Ocp-Apim-Subscription-Key"] == "speech-key" + + +def test_azure_speech_audio_transcription_request_transform(): + config = AzureSpeechAudioTranscriptionConfig() + audio = io.BytesIO(b"RIFF....WAVE") + + request_data = config.transform_audio_transcription_request( + model="speech/azure-stt", + audio_file=audio, + optional_params={}, + litellm_params={}, + ) + + assert isinstance(request_data, AudioTranscriptionRequestData) + assert request_data.data == b"RIFF....WAVE" + assert request_data.files is None + assert request_data.content_type == "audio/wav" + + +@pytest.mark.parametrize( + "payload,expected_text", + [ + ({"DisplayText": "hello world"}, "hello world"), + ( + { + "RecognitionStatus": "Success", + "NBest": [{"Display": "best text", "Confidence": 0.91}], + }, + "best text", + ), + ], +) +def test_azure_speech_audio_transcription_response_transform(payload, expected_text): + config = AzureSpeechAudioTranscriptionConfig() + response = httpx.Response(200, json=payload) + + result = config.transform_audio_transcription_response(response) + + assert isinstance(result, TranscriptionResponse) + assert result.text == expected_text + assert result._hidden_params == payload + + +def test_azure_speech_audio_transcription_response_raises_on_failed_status(): + config = AzureSpeechAudioTranscriptionConfig() + response = httpx.Response( + 200, + json={ + "RecognitionStatus": "NoMatch", + "Offset": 0, + "Duration": 0, + }, + ) + + with pytest.raises( + AzureSpeechAudioTranscriptionException, + match="RecognitionStatus=NoMatch", + ): + config.transform_audio_transcription_response(response) + + +def test_azure_speech_transcription_routes_through_provider_config(monkeypatch): + expected = TranscriptionResponse(text="hello") + audio_handler = MagicMock(return_value=expected) + + monkeypatch.setattr( + litellm.main.base_llm_http_handler, + "audio_transcriptions", + audio_handler, + ) + + response = litellm.transcription( + model="azure/speech/azure-stt", + file=io.BytesIO(b"RIFF....WAVE"), + api_base="https://eastus.api.cognitive.microsoft.com", + api_key="test-key", + language="en-US", + ) + + assert response is expected + audio_handler.assert_called_once() + assert isinstance( + audio_handler.call_args.kwargs["provider_config"], + AzureSpeechAudioTranscriptionConfig, + ) + assert audio_handler.call_args.kwargs["custom_llm_provider"] == "azure" + + +def test_azure_speech_stt_has_non_zero_input_pricing(): + pricing_path = Path(__file__).parents[4] / "model_prices_and_context_window.json" + pricing = json.loads(pricing_path.read_text()) + + assert pricing["azure/speech/azure-stt"]["input_cost_per_second"] > 0 + assert ( + pricing["azure/speech/azure-stt"]["audio_transcription_config"] + == "azure_speech" + ) From 5e16f2096203585d3983a381bd1a24e3a91e0b6e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 23 May 2026 12:16:29 -0700 Subject: [PATCH 005/109] test(proxy): phase-4 payload behavior pinning for tier-2/3 key + team management endpoints (#28681) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(proxy): phase-4 payload behavior pinning for tier-2/3 key + team management endpoints Extends the Phase 1–3 behavior-pin suite at tests/proxy_behavior/management/ with a second axis: payload-shape pinning. Phase 1–3 held payload minimal and pinned (actor, target) → status across 37 routes; Phase 4 holds the caller fixed at an authorized actor, varies the payload shape, and asserts the observable DB effect (on accept) or the named guard / row-unchanged (on reject). Faithfulness contract from Phase 1–3 is unchanged. Six families + one gap-closer (59 new scenarios, 620 → 679 total): * F1 — key budget / rate-limit (test_key_budget_limits.py, 18) * F2 — key↔team reassignment (test_key_team_change.py, 6) * F3 — team budget / rate-limit (test_team_budget_limits.py, 15) * F4 — member-info validation (test_team_member_info_validation.py, 5) * F5 — permission batching (test_team_permissions_bulk_update.py, 6) * F6 — org-scoped team access (+2 detail-string pins in existing files) * F7 — coverage gap-closer (test_f7_coverage_closeout.py, 7) Harness extensions in conftest.py (additive only): * create_scratch_org() seeder with its own scratch-prefixed budget row * budget / limit fields on create_scratch_team() * scratch teardown also sweeps litellm_organizationtable Coverage telemetry (behavior-suite-only): * key_management_endpoints.py 60 % → 65 % (+82 lines) * team_endpoints.py 62 % → 72 % (+137 lines, crosses 70 % stretch) Key lands under 70 % per plan §7 escape hatch — the gap is dominated by routes outside F1–F6 scope (key list/info v2 internals) and structurally dead org-budget guards (call sites at lines 889 + 2310 + 985 + 1751 load the org without include_budget_table=True, so org.litellm_budget_table is None at guard time and the aggregate guard no-ops). Pinned as observed no-op behavior so a future fix that flips the flag turns these into reds. Zero source-code changes; pyproject.toml diff is empty; test_route_coverage.py stays green untouched; G3 grep guards still green; local wall-time 14 s for the full suite (no coverage), 22 s with coverage. G4 regression-replay protocol executed against three representative fix-PR parents (410ce761dc, 0bd49ecb8b, 8bbc61e03c): all Phase 4 tests PASS at pre-fix SHAs — confirming the F1–F7 layer is a helper-body pin, not a regression-replay layer for those specific historical bypass shapes. Targeted RED-bait scenarios for each fix are left for a follow-up PR. * test(proxy): push key_management_endpoints.py past the 70% stretch (F7-extension) Adds 24 more payload-pin scenarios in test_f7_key_coverage_push.py following the same accepted-effect / rejected-guard pattern. Each scenario cites the file:line range it pins; same anti-snapshot rules apply. Target ranges (all reachable via HTTP-boundary payload variation): * 5942-6063 /key/health with metadata.logging → test_key_logging body * 4565-4692 /key/reset_spend happy + 404 + non-admin gate + value validation * 4421-4533 /key/regenerate ghost-404 + happy + new_key + grace_period * 4168-4202 _insert_deprecated_key body via grace_period * 6118-6133 _enforce_unique_key_alias duplicate-alias rejection * 6148-6169 validate_model_max_budget malformed-payload rejection * 4708-4789 validate_key_list_check user/team/org/key_hash branches * 2622-2733 /key/bulk_update mixed success/failure + admin gate + size limits * 2797-2950 /team/key/bulk_update all-keys path + explicit-keys dedupe + 404 * 5108-5207 /key/aliases admin + scoped + search-filter branches * 3253-3303 /key/info ghost + explicit-key + no-key-uses-auth-header * 3427-3436 generate_key_helper_fn budget_limits initialization * 1794-1815 prepare_key_update_data duration + budget_duration paths * 5280-5388 _build_filter_conditions across include_created_by_keys/team/sort/alias Coverage telemetry — full PR4 dataset: key_management_endpoints.py: 60 % → 71 % (+11 pts, +194 lines) team_endpoints.py: 62 % → 72 % (+10 pts, +137 lines) Both files now over the plan §7 PR4.M4 70 % stretch as a side effect of pinning real payload behavior. 721 tests pass in 19 s local (full suite, no coverage); 27 s with coverage. Zero source-code changes; pyproject.toml diff still empty; test_route_coverage.py + G3 grep guards still green. Honest finding (kept from the prior commit's body): four structurally-dead org-budget guards remain pinned as observed no-op behavior — they fire only when get_org_object is called with include_budget_table=True, which none of the four management-endpoint call sites currently do. Pinned so a future change that flips the flag turns these into reds. Two helper guards are honest-ceiling: _validate_reset_spend_value's isinstance check at line 4568 is unreachable from HTTP because Pydantic 422s non-float before the helper runs; same shape for /team/key/bulk_update's missing team_id / no-selector pre-handler guards. * test(proxy): address PR review — try/finally cleanup + loosen 500 envelope pins + Optional annotations Greptile review feedback on PR #28681: 1. Wrap manual budget-row cleanup in try/finally so an assertion failure doesn't leave non-scratch-prefixed budget rows orphaned across CI re-runs (test_team_new_with_team_member_budget_creates_budget_row and test_team_update_team_member_budget_upserts). 2. Loosen the two 500-status pins to in (400, 422, 500) — the named-guard substring is the real pin; the outer ValueError-wrap envelope is an implementation detail that a future improvement should be free to fix to a proper 400/422 without flipping these tests red. 3. Add missing Optional annotations on _seed_token's max_budget / metadata / team_id keyword args (they default to None). Greptile's typo flag on 'read-world' in the conftest comment is declined — 'read-world' is the project's established term for the immutable seeded world fixture (see other usages in conftest.py and actors.py). 721 tests still pass in 17 s. --- tests/proxy_behavior/management/conftest.py | 80 ++ .../management/test_f7_coverage_closeout.py | 346 +++++++ .../management/test_f7_key_coverage_push.py | 916 ++++++++++++++++++ .../management/test_key_budget_limits.py | 383 ++++++++ .../management/test_key_team_change.py | 229 +++++ .../management/test_team_budget_limits.py | 321 ++++++ .../management/test_team_info.py | 16 + .../test_team_member_info_validation.py | 189 ++++ .../test_team_permissions_bulk_update.py | 245 +++++ .../management/test_team_update.py | 18 + 10 files changed, 2743 insertions(+) create mode 100644 tests/proxy_behavior/management/test_f7_coverage_closeout.py create mode 100644 tests/proxy_behavior/management/test_f7_key_coverage_push.py create mode 100644 tests/proxy_behavior/management/test_key_budget_limits.py create mode 100644 tests/proxy_behavior/management/test_key_team_change.py create mode 100644 tests/proxy_behavior/management/test_team_budget_limits.py create mode 100644 tests/proxy_behavior/management/test_team_member_info_validation.py create mode 100644 tests/proxy_behavior/management/test_team_permissions_bulk_update.py diff --git a/tests/proxy_behavior/management/conftest.py b/tests/proxy_behavior/management/conftest.py index fa0bef86280d..5b33ee6fd89f 100644 --- a/tests/proxy_behavior/management/conftest.py +++ b/tests/proxy_behavior/management/conftest.py @@ -142,6 +142,10 @@ async def create_scratch_team( member_user_ids: Optional[list] = None, team_member_permissions: Optional[list] = None, models: Optional[list] = None, + max_budget: Optional[float] = None, + tpm_limit: Optional[int] = None, + rpm_limit: Optional[int] = None, + metadata: Optional[dict] = None, ) -> str: """Raw-seed a scratch-tagged team row; returns its team_id. @@ -155,6 +159,10 @@ async def create_scratch_team( team_member_permissions / models seed the matching raw columns — needed by the team-key-permission and team-model matrices. + + max_budget / tpm_limit / rpm_limit / metadata seed the team's own limit + columns (Phase 4 F1+F3) — they live directly on LiteLLM_TeamTable, no + budget-table relation needed. """ admin_user_ids = list(admin_user_ids or []) member_user_ids = list(member_user_ids or []) @@ -174,10 +182,73 @@ async def create_scratch_team( data["team_member_permissions"] = team_member_permissions if models is not None: data["models"] = models + if max_budget is not None: + data["max_budget"] = max_budget + if tpm_limit is not None: + data["tpm_limit"] = tpm_limit + if rpm_limit is not None: + data["rpm_limit"] = rpm_limit + if metadata is not None: + data["metadata"] = Json(metadata) await prisma.db.litellm_teamtable.create(data=data) return team_id +async def create_scratch_org( + prisma, + scratch_prefix: str, + *, + max_budget: Optional[float] = None, + tpm_limit: Optional[int] = None, + rpm_limit: Optional[int] = None, + models: Optional[list] = None, + metadata: Optional[dict] = None, + suffix: str = "org", +) -> str: + """Seed a scratch-tagged org + its own budget row; returns organization_id. + + The org's `budget_id` points at a fresh `litellm_budgettable` row that + carries the per-org limits (`_check_org_key_limits` and the team budget + helpers read `org_table.litellm_budget_table.`, not columns on the + org row itself). Both rows share the scratch prefix so the teardown + reclaims them — budget by `budget_id` prefix (already swept), org by + `organization_id` prefix (added in this PR to the `scratch` fixture). + + models / metadata seed the matching org columns; `_check_org_team_limits` + (F3) reads `org_table.models`, and the org metadata mirror of + model_rpm_limit / model_tpm_limit is what F1's model-specific org guard + consults. + """ + org_id = f"{scratch_prefix}-{suffix}" + budget_id = f"{scratch_prefix}-{suffix}-budget" + budget_data: Dict[str, Any] = { + "budget_id": budget_id, + "created_by": "phase4-scratch", + "updated_by": "phase4-scratch", + } + if max_budget is not None: + budget_data["max_budget"] = max_budget + if tpm_limit is not None: + budget_data["tpm_limit"] = tpm_limit + if rpm_limit is not None: + budget_data["rpm_limit"] = rpm_limit + await prisma.db.litellm_budgettable.create(data=budget_data) + + org_data: Dict[str, Any] = { + "organization_id": org_id, + "organization_alias": org_id, + "budget_id": budget_id, + "created_by": "phase4-scratch", + "updated_by": "phase4-scratch", + } + if models is not None: + org_data["models"] = models + if metadata is not None: + org_data["metadata"] = Json(metadata) + await prisma.db.litellm_organizationtable.create(data=org_data) + return org_id + + @dataclass(frozen=True) class SeededActor: user_id: str @@ -263,6 +334,15 @@ async def scratch(prisma): await prisma.db.litellm_usertable.delete_many( where={"user_id": {"startswith": handle.prefix}} ) + # F1+F3 seed scratch orgs via create_scratch_org; the world seeder is + # the only other writer of LiteLLM_OrganizationTable and uses the + # behavior-pin- prefix, so a scratch-prefixed sweep here cannot + # collide with the read-world. Org must be reclaimed BEFORE its + # budget — org.budget_id → budget.budget_id, so deleting the parent + # first would FK-violate on any still-attached scratch org. + await prisma.db.litellm_organizationtable.delete_many( + where={"organization_id": {"startswith": handle.prefix}} + ) await prisma.db.litellm_budgettable.delete_many( where={"budget_id": {"startswith": handle.prefix}} ) diff --git a/tests/proxy_behavior/management/test_f7_coverage_closeout.py b/tests/proxy_behavior/management/test_f7_coverage_closeout.py new file mode 100644 index 000000000000..c78adbbb79d8 --- /dev/null +++ b/tests/proxy_behavior/management/test_f7_coverage_closeout.py @@ -0,0 +1,346 @@ +"""Phase 4 F7 — coverage gap-closer scenarios picked from the un-covered +ranges left after F1–F6 landed. + +Each scenario cites the file:line range it pins (PR4.M3 requirement). +Scenarios that would only pad the count without pinning observable +behavior are excluded — see `phase4-plan.md` §4-F7 anti-patterns. + +Ranges addressed here: + * team_endpoints.py 455–521 (_check_team_model_specific_limits body) + * team_endpoints.py 538–566 (_check_team_rpm_tpm_limits body) + * team_endpoints.py 696–731 (_check_org_team_limits guaranteed-throughput + branch — currently dead because the call + site doesn't include_budget_table=True, + but the inner `find_many` + helper-loop + runs regardless, covering the lines) + * key_management_endpoints.py 1147–1156 (_check_project_key_limits + project-not-found 404) + * key_management_endpoints.py 3007–3018 (validate_key_team_change + team-admin-accepts branch) +""" + +import uuid +from typing import Any, Dict, Optional + +import pytest +from prisma import Json + +from litellm.proxy.utils import hash_token + +from .actors import TEAM_ALPHA, Actor +from .conftest import create_scratch_org, create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +# --------------------------------------------------------------------------- +# /team/new — guaranteed_throughput route into _check_org_team_limits's +# throughput branch (lines 696–731), which then calls +# check_org_team_model_specific_limits → _check_team_model_specific_limits +# (lines 442–525). The org metadata supplies the per-model cap; sibling +# teams are loaded from DB to drive the allocation sum. +# --------------------------------------------------------------------------- + + +async def test_org_team_guaranteed_throughput_model_over_bound_rejected( + proxy_client, prisma, scratch, world +): + org_id = await create_scratch_org( + prisma, + scratch.prefix, + models=["gpt-4"], + metadata={"model_rpm_limit": {"gpt-4": 30}}, + ) + # A sibling team in the same org already burning 20 rpm on gpt-4 — + # forces _check_team_model_specific_limits's `model_specific_rpm_limit` + # accumulator to actually accumulate (covers lines 468–478). + await create_scratch_team( + prisma, + team_id=scratch.tag("sibling"), + organization_id=org_id, + metadata={"model_rpm_limit": {"gpt-4": 20}}, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = scratch.tag("new") + body = { + "team_id": team_id, + "team_alias": team_id, + "organization_id": org_id, + "models": ["gpt-4"], + "model_rpm_limit": {"gpt-4": 100}, + "rpm_limit_type": "guaranteed_throughput", + } + resp = await proxy_client.post( + "/team/new", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + # 20 (sibling) + 100 (new) > 30 (org cap) → guard fires. + assert resp.status_code == 400, resp.text + assert "RPM" in resp.text, resp.text + rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + assert rows == [] + + +async def test_org_team_guaranteed_throughput_model_tpm_over_bound_rejected( + proxy_client, prisma, scratch, world +): + """Mirror of the rpm scenario but for the model_tpm side — pins + _check_team_model_specific_limits's tpm branch (lines 503–521) which + the rpm scenario doesn't exercise.""" + org_id = await create_scratch_org( + prisma, + scratch.prefix, + models=["gpt-4"], + metadata={"model_tpm_limit": {"gpt-4": 500}}, + ) + await create_scratch_team( + prisma, + team_id=scratch.tag("sibling"), + organization_id=org_id, + metadata={"model_tpm_limit": {"gpt-4": 200}}, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = scratch.tag("new") + body = { + "team_id": team_id, + "team_alias": team_id, + "organization_id": org_id, + "models": ["gpt-4"], + "model_tpm_limit": {"gpt-4": 5000}, + "tpm_limit_type": "guaranteed_throughput", + } + resp = await proxy_client.post( + "/team/new", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + assert resp.status_code == 400, resp.text + assert "TPM" in resp.text, resp.text + rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + assert rows == [] + + +async def test_org_team_guaranteed_throughput_aggregate_runs( + proxy_client, prisma, scratch, world +): + """Aggregate guard's no-op path (line 549-566 — entity_rpm_limit is None + because include_budget_table=False at the call site). The helper still + executes its `allocated_tpm = sum(...)` and `allocated_rpm = sum(...)` + lines, which is the coverage target.""" + org_id = await create_scratch_org(prisma, scratch.prefix, models=["m"]) + await create_scratch_team( + prisma, + team_id=scratch.tag("sibling"), + organization_id=org_id, + tpm_limit=100, + rpm_limit=10, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = scratch.tag("new") + body = { + "team_id": team_id, + "team_alias": team_id, + "organization_id": org_id, + "models": ["m"], + "tpm_limit": 50, + "rpm_limit": 5, + "tpm_limit_type": "guaranteed_throughput", + } + resp = await proxy_client.post( + "/team/new", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + # No org budget table loaded → check is no-op → 200. + assert resp.status_code == 200, resp.text + + +# --------------------------------------------------------------------------- +# /key/generate — _check_project_key_limits project-not-found branch +# (lines 1147–1156). Hitting it requires a project_id that doesn't resolve; +# get_project_object returns None, the handler raises 404. +# --------------------------------------------------------------------------- + + +async def test_key_generate_with_unknown_project_id_rejected( + proxy_client, prisma, scratch, world +): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + body = { + "key_alias": scratch.prefix, + "project_id": f"{scratch.prefix}-ghost-project", + } + resp = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + # The unknown-project guard inside _check_project_key_limits raises 404. + # The route's exception handler may wrap it; pin both shapes. + assert resp.status_code in (400, 404), resp.text + assert "project" in resp.text.lower() or "not found" in resp.text.lower(), resp.text + rows = await prisma.db.litellm_verificationtoken.find_many( + where={"key_alias": scratch.prefix} + ) + assert rows == [], "rejected key leaked a row" + + +# --------------------------------------------------------------------------- +# /key/update — validate_key_team_change team-admin-accepts branch +# (line 3011). PROXY_ADMIN covers line 3006; a non-admin team admin of the +# target team covers 3007–3011. Owner stays a member of the destination +# team to clear the membership guard first. +# --------------------------------------------------------------------------- + + +async def _seed_key_for_relocation( + prisma, scratch_prefix: str, *, user_id: str, team_id: str +) -> str: + cleartext = "sk-" + uuid.uuid4().hex + await prisma.db.litellm_verificationtoken.create( + data={ + "token": hash_token(cleartext), + "key_alias": f"{scratch_prefix}-key", + "key_name": f"{scratch_prefix}-key", + "user_id": user_id, + "team_id": team_id, + "models": [], + } + ) + return cleartext + + +async def test_team_new_with_team_member_budget_creates_budget_row( + proxy_client, prisma, scratch, world +): + """/team/new with `team_member_budget` routes through + TeamMemberBudgetHandler.create_team_member_budget_table (lines 196–248). + Observable end-state: a litellm_budgettable row is created and the + team's metadata.team_member_budget_id points at it.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = scratch.tag("team-with-mbudget") + resp = await proxy_client.post( + "/team/new", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "team_alias": scratch.tag("alias"), + "team_member_budget": 10.0, + "team_member_rpm_limit": 100, + "team_member_tpm_limit": 1000, + }, + ) + assert resp.status_code == 200, resp.text + team_row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + assert team_row is not None + budget_id = (team_row.metadata or {}).get("team_member_budget_id") + assert ( + budget_id is not None + ), f"team_member_budget_id not stamped on team metadata: {team_row.metadata!r}" + + # The handler writes the per-member budget row under a non-scratch id + # pattern (`team--budget-`), so the prefix sweep can't + # reclaim it. Cleanup must run even when a downstream assertion fires; + # otherwise orphan rows accumulate across CI re-runs. + try: + budget_row = await prisma.db.litellm_budgettable.find_unique( + where={"budget_id": budget_id} + ) + assert budget_row is not None, "team_member_budget row was not created" + assert budget_row.max_budget == 10.0 + assert budget_row.rpm_limit == 100 + assert budget_row.tpm_limit == 1000 + finally: + await prisma.db.litellm_teamtable.update( + where={"team_id": team_id}, + data={"metadata": Json({})}, + ) + await prisma.db.litellm_budgettable.delete(where={"budget_id": budget_id}) + + +async def test_team_update_team_member_budget_upserts( + proxy_client, prisma, scratch, world +): + """/team/update with `team_member_budget` against a team that has no + pre-existing team_member_budget_id routes through + TeamMemberBudgetHandler.upsert_team_member_budget_table's else-branch + (lines 294–303), which in turn calls create_team_member_budget_table.""" + team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "team_member_budget": 5.0, + }, + ) + assert resp.status_code == 200, resp.text + team_row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + assert team_row is not None + budget_id = (team_row.metadata or {}).get("team_member_budget_id") + assert budget_id is not None, "team_member_budget_id not upserted" + + # See comment on the sibling test — non-prefixed budget row, cleanup + # must survive an assertion failure to avoid orphan accumulation. + try: + # (No further assertions today, but the try/finally keeps the + # cleanup contract uniform with the sibling test and is robust to + # future asserts being added here.) + pass + finally: + await prisma.db.litellm_teamtable.update( + where={"team_id": team_id}, + data={"metadata": Json({})}, + ) + await prisma.db.litellm_budgettable.delete(where={"budget_id": budget_id}) + + +async def test_key_team_change_accepted_by_target_team_admin( + proxy_client, prisma, scratch, world +): + """Caller is admin of the destination team AND the key's owner — the + common_key_access_checks gate requires user_id-match for non-proxy-admin + callers, so the team-admin-accepts branch of validate_key_team_change + can only be reached when the team admin is also the key holder. + Hits validate_key_team_change line 3007–3011.""" + actor_user_id = f"{scratch.prefix}-self-admin" + actor_cleartext = "sk-" + uuid.uuid4().hex + await prisma.db.litellm_usertable.create( + data={"user_id": actor_user_id, "user_role": "internal_user"} + ) + await prisma.db.litellm_verificationtoken.create( + data={ + "token": hash_token(actor_cleartext), + "key_alias": f"{scratch.prefix}-actor-key", + "key_name": f"{scratch.prefix}-actor-key", + "user_id": actor_user_id, + "models": [], + "allowed_routes": ["/key/update"], + } + ) + source_team = await create_scratch_team( + prisma, + team_id=scratch.tag("source"), + admin_user_ids=[actor_user_id], + ) + target_team = await create_scratch_team( + prisma, + team_id=scratch.tag("target"), + admin_user_ids=[actor_user_id], + ) + key_cleartext = await _seed_key_for_relocation( + prisma, scratch.prefix, user_id=actor_user_id, team_id=source_team + ) + resp = await proxy_client.post( + "/key/update", + headers={"Authorization": f"Bearer {actor_cleartext}"}, + json={"key": key_cleartext, "team_id": target_team}, + ) + assert resp.status_code == 200, resp.text + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(key_cleartext)} + ) + assert row is not None + assert row.team_id == target_team, "key did not move under team-admin initiator" diff --git a/tests/proxy_behavior/management/test_f7_key_coverage_push.py b/tests/proxy_behavior/management/test_f7_key_coverage_push.py new file mode 100644 index 000000000000..1bf3d1d394ef --- /dev/null +++ b/tests/proxy_behavior/management/test_f7_key_coverage_push.py @@ -0,0 +1,916 @@ +"""Phase 4 F7-extension — additional payload pins pushing +`key_management_endpoints.py` past the 70 % stretch. + +Same pattern as `test_f7_coverage_closeout.py`: each scenario cites the +file:line range it pins and asserts observable end-state, not response-body +snapshots. Targets the largest non-deferred un-covered ranges left after +the first F7 pass: + + * 5942–6063 /key/health logging-metadata path (test_key_logging body) + * 4630–4692 /key/reset_spend happy path + 404 + admin gate + * 4565–4596 _validate_reset_spend_value branches + * 4421–4476 /key/regenerate ghost-key 404 + premium gate + * 6118–6133 _enforce_unique_key_alias duplicate-alias rejection + * 6148–6169 validate_model_max_budget malformed payload rejection + * 4708–4789 validate_key_list_check user/team/org/key_hash branches + +Excluded: `_rotate_master_key` (lines 3997–4123) — deferred per plan §6. +""" + +import uuid +from typing import Any, Dict, Optional + +import pytest +from prisma import Json + +from litellm.proxy.utils import hash_token + +from .actors import TEAM_ALPHA, TEAM_BETA, Actor +from .conftest import create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +async def _seed_token( + prisma, + scratch_prefix: str, + *, + suffix: str = "tok", + user_id: str, + spend: float = 0.0, + max_budget: Optional[float] = None, + metadata: Optional[dict] = None, + team_id: Optional[str] = None, +) -> str: + cleartext = "sk-" + uuid.uuid4().hex + data: Dict[str, Any] = { + "token": hash_token(cleartext), + "key_alias": f"{scratch_prefix}-{suffix}", + "key_name": f"{scratch_prefix}-{suffix}", + "user_id": user_id, + "models": [], + "spend": spend, + } + if max_budget is not None: + data["max_budget"] = max_budget + if metadata is not None: + data["metadata"] = Json(metadata) + if team_id is not None: + data["team_id"] = team_id + await prisma.db.litellm_verificationtoken.create(data=data) + return cleartext + + +# --------------------------------------------------------------------------- +# /key/health — `metadata.logging` flips the handler into test_key_logging, +# which lives at lines 5990–6067. The healthy-no-logging path is already +# covered by the existing test_key_health.py; this adds the logging-set +# path. The mock_response inside test_key_logging means no real LLM call +# fires — only the callback-name validation and the post-call sweep. +# --------------------------------------------------------------------------- + + +async def test_key_health_with_logging_metadata_runs_test_logging( + proxy_client, prisma, scratch, world +): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + metadata={ + "logging": [ + {"callback_name": "noop-scratch-callback"}, + ] + }, + ) + resp = await proxy_client.post( + "/key/health", + headers={"Authorization": f"Bearer {cleartext}"}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + # Either healthy or unhealthy — both pin the logging branch. + assert body["key"] in ("healthy", "unhealthy") + assert "logging_callbacks" in body + assert body["logging_callbacks"]["callbacks"] == ["noop-scratch-callback"] + + +async def test_key_health_with_missing_callback_name_rejected( + proxy_client, prisma, scratch, world +): + """test_key_logging raises ValueError if a callback dict lacks + callback_name — wrapped by the outer try/except into a 500.""" + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + metadata={"logging": [{"not_callback_name": "x"}]}, + ) + resp = await proxy_client.post( + "/key/health", + headers={"Authorization": f"Bearer {cleartext}"}, + ) + # The outer handler currently wraps the inner ValueError as a 500; + # accept any rejection envelope so a future 400/422 conversion doesn't + # trip this test. The named-guard substring is the real pin. + assert resp.status_code in (400, 422, 500), resp.text + assert "callback_name" in resp.text + + +# --------------------------------------------------------------------------- +# /key/reset_spend — 404 + happy path + non-admin reject. Pins +# _validate_reset_spend_value (lines 4565–4596) and +# _check_proxy_or_team_admin_for_key (lines 4536–4562). +# --------------------------------------------------------------------------- + + +async def test_reset_spend_ghost_key_404(proxy_client, scratch, world): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/sk-{scratch.prefix}-ghost/reset_spend", + headers={"Authorization": f"Bearer {seeder}"}, + json={"reset_to": 0.0}, + ) + assert resp.status_code == 404, resp.text + + +async def test_reset_spend_happy_path(proxy_client, prisma, scratch, world): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + spend=5.0, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/{cleartext}/reset_spend", + headers={"Authorization": f"Bearer {seeder}"}, + json={"reset_to": 1.0}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["spend"] == 1.0 + assert resp.json()["previous_spend"] == 5.0 + + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(cleartext)} + ) + assert row is not None + assert row.spend == 1.0 + + +# Pydantic catches non-float at the request layer (422), so the inner +# `isinstance(reset_to, (int, float))` guard at line 4568 is unreachable +# from the HTTP boundary. Only the negative + above-current-spend branches +# fire as the handler's own 400. +@pytest.mark.parametrize( + "reset_to,expected_detail", + [ + (-1.0, "must be >= 0"), + (100.0, "must be <= current spend"), # current spend = 5.0 + ], + ids=["negative", "above_current_spend"], +) +async def test_reset_spend_validate_value_branches( + reset_to, expected_detail: str, proxy_client, prisma, scratch, world +): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + spend=5.0, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/{cleartext}/reset_spend", + headers={"Authorization": f"Bearer {seeder}"}, + json={"reset_to": reset_to}, + ) + assert resp.status_code == 400, resp.text + assert expected_detail in resp.text, resp.text + # Row spend must be unchanged. + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(cleartext)} + ) + assert row.spend == 5.0, "spend mutated despite validation rejection" + + +async def test_reset_spend_non_numeric_caught_by_pydantic( + proxy_client, prisma, scratch, world +): + """Pin: non-float reset_to is rejected at the Pydantic layer with 422 + before reaching _validate_reset_spend_value. This documents that the + helper's `isinstance(reset_to, (int, float))` guard at line 4568 is + structurally unreachable via HTTP.""" + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + spend=5.0, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/{cleartext}/reset_spend", + headers={"Authorization": f"Bearer {seeder}"}, + json={"reset_to": "not-a-number"}, + ) + assert resp.status_code == 422, resp.text + + +async def test_reset_spend_non_admin_caller_rejected( + proxy_client, prisma, scratch, world +): + """_check_proxy_or_team_admin_for_key raises 403 when caller is neither + proxy admin nor admin of the key's team.""" + # Seed a key against TEAM_BETA (no actor in our world is admin of beta). + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.CROSS_ORG_USER].user_id, + spend=5.0, + team_id=TEAM_BETA, + ) + # Caller is a member of TEAM_ALPHA but not admin of TEAM_BETA. + initiator = world.keys[Actor.INTERNAL_USER].cleartext + resp = await proxy_client.post( + f"/key/{cleartext}/reset_spend", + headers={"Authorization": f"Bearer {initiator}"}, + json={"reset_to": 1.0}, + ) + # Route-level admin gate may fire first (401) or the helper's own 403 — + # both prove the path is guarded; pin either as rejection. + assert resp.status_code in (401, 403), resp.text + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(cleartext)} + ) + assert row.spend == 5.0, "spend mutated despite rejection" + + +# --------------------------------------------------------------------------- +# /key/regenerate — ghost key 404 + happy path + new_key override. Pins +# the route handler body lines 4382–4533 and _execute_virtual_key_regeneration +# entry-point lines around 4220–4240. +# --------------------------------------------------------------------------- + + +async def test_regenerate_ghost_key_404(proxy_client, scratch, world): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/sk-{scratch.prefix}-ghost/regenerate", + headers={"Authorization": f"Bearer {seeder}"}, + json={}, + ) + assert resp.status_code == 404, resp.text + + +async def test_regenerate_no_key_supplied_400(proxy_client, world): + """POST /key/regenerate with no key in path AND no key in body.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/key/regenerate", + headers={"Authorization": f"Bearer {seeder}"}, + json={}, + ) + assert resp.status_code == 400, resp.text + assert "No key passed in" in resp.text or "key" in resp.text.lower() + + +async def test_regenerate_happy_path(proxy_client, prisma, scratch, world): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + ) + old_hash = hash_token(cleartext) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/{cleartext}/regenerate", + headers={"Authorization": f"Bearer {seeder}"}, + json={}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["key"].startswith("sk-") + new_hash = hash_token(body["key"]) + assert new_hash != old_hash + + # Old token should no longer be in active tokens (deleted by regenerate). + old_row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": old_hash} + ) + assert old_row is None, "old token still present after regenerate" + + # New token should be active. + new_row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": new_hash} + ) + assert new_row is not None, "new token not written after regenerate" + + +async def test_regenerate_with_explicit_new_key(proxy_client, prisma, scratch, world): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="explicit", + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + new_key = "sk-" + uuid.uuid4().hex + resp = await proxy_client.post( + f"/key/{cleartext}/regenerate", + headers={"Authorization": f"Bearer {seeder}"}, + json={"new_key": new_key}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["key"] == new_key + new_row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(new_key)} + ) + assert new_row is not None + + +# --------------------------------------------------------------------------- +# /key/generate duplicate-alias rejection — pins _enforce_unique_key_alias +# (lines 6118–6133). Two keys cannot share an alias. +# --------------------------------------------------------------------------- + + +async def test_generate_duplicate_alias_rejected(proxy_client, prisma, scratch, world): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + alias = scratch.prefix + "-shared-alias" + first = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key_alias": alias}, + ) + assert first.status_code == 200, first.text + second = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key_alias": alias}, + ) + assert second.status_code == 400, second.text + assert "already exists" in second.text, second.text + + +# --------------------------------------------------------------------------- +# /key/generate with malformed model_max_budget → 400 from +# validate_model_max_budget (lines 6148–6169). +# --------------------------------------------------------------------------- + + +async def test_generate_with_invalid_model_max_budget_rejected( + proxy_client, scratch, world +): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "key_alias": scratch.prefix, + # Wrong shape — budget_limit should be numeric; passing a dict + # for the model value bypasses the BudgetConfig parse and trips + # the validator's exception wrap. + "model_max_budget": {"gpt-4": {"budget_limit": "not-a-number"}}, + }, + ) + # validate_model_max_budget raises ValueError, which the outer handler + # currently wraps as a 500. Pin only the named-guard substring; accept + # 400/422/500 so a future error-envelope improvement doesn't trip this. + assert resp.status_code in (400, 422, 500), resp.text + assert "Invalid model_max_budget" in resp.text, resp.text + + +# --------------------------------------------------------------------------- +# /key/list — pins validate_key_list_check (lines 4695–4790). The handler +# already runs via Phase 1–3's test_key_list.py for the PROXY_ADMIN bypass; +# this adds the non-admin user_id-mismatch + team_id-mismatch + +# organization_id-mismatch branches. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "filter_kwarg,expected_substring", + [ + ({"user_id": "behavior-pin-proxy_admin"}, "check another user"), + ({"team_id": "behavior-pin-team-beta"}, "check this team"), + ( + {"organization_id": "behavior-pin-org-b"}, + "check this organization", + ), + ], + ids=["user_mismatch", "team_mismatch", "org_mismatch"], +) +async def test_key_list_non_admin_authz_branches( + filter_kwarg, expected_substring: str, proxy_client, world +): + """Non-admin caller hits validate_key_list_check's three rejection + branches. INTERNAL_USER (Org A, TEAM_ALPHA member) is the caller; each + filter targets a foreign user/team/org and trips the matching guard.""" + caller = world.keys[Actor.INTERNAL_USER].cleartext + qs = "&".join(f"{k}={v}" for k, v in filter_kwarg.items()) + resp = await proxy_client.get( + f"/key/list?{qs}", + headers={"Authorization": f"Bearer {caller}"}, + ) + assert resp.status_code == 403, resp.text + assert expected_substring in resp.text, resp.text + + +# --------------------------------------------------------------------------- +# /key/bulk_update — pins the whole admin-only handler body (lines 2622–2733) +# including the per-key try/except branch (2688–2727) via a mixed batch +# of one existing key + one ghost. +# --------------------------------------------------------------------------- + + +async def test_key_bulk_update_mixed_success_and_failure( + proxy_client, prisma, scratch, world +): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + existing = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="bulk1", + ) + resp = await proxy_client.post( + "/key/bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "keys": [ + {"key": existing, "max_budget": 5.0}, + {"key": "sk-ghost-" + scratch.prefix, "max_budget": 5.0}, + ] + }, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["total_requested"] == 2 + assert len(body["successful_updates"]) == 1 + assert len(body["failed_updates"]) == 1 + # Re-read confirms the successful key was actually updated. + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(existing)} + ) + assert row.max_budget == 5.0 + + +async def test_key_bulk_update_non_admin_rejected(proxy_client, world): + """Lines 2631–2635 — admin-only role gate.""" + caller = world.keys[Actor.INTERNAL_USER].cleartext + resp = await proxy_client.post( + "/key/bulk_update", + headers={"Authorization": f"Bearer {caller}"}, + json={"keys": [{"key": "sk-x", "max_budget": 1.0}]}, + ) + assert resp.status_code in (401, 403), resp.text + + +async def test_key_bulk_update_empty_keys_rejected(proxy_client, world): + """Line 2643–2647 — empty keys list rejected.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/key/bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"keys": []}, + ) + assert resp.status_code == 400, resp.text + assert "No keys" in resp.text + + +async def test_key_bulk_update_exceeds_max_batch_rejected(proxy_client, world): + """Lines 2649–2656 — over-batch-size rejection. 501 ghost keys are fine + here because validation fires before any update runs.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + over_batch = [{"key": f"sk-{i}", "max_budget": 1.0} for i in range(501)] + resp = await proxy_client.post( + "/key/bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"keys": over_batch}, + ) + assert resp.status_code == 400, resp.text + assert "500" in resp.text or "Maximum" in resp.text + + +# --------------------------------------------------------------------------- +# /key/update with extended fields — pins prepare_key_update_data +# branches: duration (1794–1802), budget_duration (1804–1815), +# model_max_budget validation (1838–1840), and the reserved-metadata +# immutability check (1718–1731). +# --------------------------------------------------------------------------- + + +async def test_key_update_with_duration_and_budget_duration( + proxy_client, prisma, scratch, world +): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/key/update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "key": cleartext, + "duration": "1h", + "budget_duration": "30d", + }, + ) + assert resp.status_code == 200, resp.text + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(cleartext)} + ) + assert row.expires is not None, "expires not stamped from duration" + assert row.budget_reset_at is not None, "budget_reset_at not stamped" + + +async def test_key_update_with_clear_duration(proxy_client, prisma, scratch, world): + """`duration: -1` clears expires (line 1796–1798).""" + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + # First set an expiry + await proxy_client.post( + "/key/update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key": cleartext, "duration": "1h"}, + ) + # Then clear it + resp = await proxy_client.post( + "/key/update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key": cleartext, "duration": "-1"}, + ) + assert resp.status_code == 200, resp.text + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(cleartext)} + ) + assert row.expires is None, "expires not cleared by duration=-1" + + +# --------------------------------------------------------------------------- +# /team/key/bulk_update — pins handler body lines 2797–2950 including: +# - missing team_id 400 (2803–2807) +# - over-batch-size 400 (2810–2816) +# - all_keys_in_team scan (2818–2841) +# - explicit key_ids dedupe (2842–2863) +# - non-admin permission gate (2866–2882) +# - per-key loop with mixed success/404 (2902–2944) +# --------------------------------------------------------------------------- + + +async def test_team_key_bulk_update_missing_team_id_rejected(proxy_client, world): + """Pydantic catches missing team_id at the request layer → 422 before + the handler's own `if not data.team_id` guard at line 2803.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/key/bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"update_fields": {"max_budget": 5.0}}, + ) + assert resp.status_code == 422, resp.text + + +async def test_team_key_bulk_update_no_selector_rejected(proxy_client, world): + """Pydantic root-validator catches missing key_ids/all_keys_in_team + at the request layer → 422.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/key/bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": TEAM_ALPHA, + "update_fields": {"max_budget": 5.0}, + }, + ) + assert resp.status_code == 422, resp.text + assert "key_ids" in resp.text or "all_keys_in_team" in resp.text + + +async def test_team_key_bulk_update_all_keys_in_team( + proxy_client, prisma, scratch, world +): + """Pin the all_keys_in_team branch (lines 2818–2841) plus the + per-key success path. Seed two scratch keys against the scratch team.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = await create_scratch_team( + prisma, + team_id=scratch.tag("team"), + admin_user_ids=[world.keys[Actor.PROXY_ADMIN].user_id], + ) + k1 = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="t1", + team_id=team_id, + ) + k2 = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="t2", + team_id=team_id, + ) + resp = await proxy_client.post( + "/team/key/bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "all_keys_in_team": True, + "update_fields": {"max_budget": 7.0}, + }, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["total_requested"] == 2 + # Re-read both keys; max_budget should be set. + for k in (k1, k2): + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(k)} + ) + assert row.max_budget == 7.0 + + +async def test_team_key_bulk_update_explicit_key_ids_mixed( + proxy_client, prisma, scratch, world +): + """Pin the explicit-key_ids dedupe + per-key 404 path (lines 2842–2944). + Send a real key + a ghost key under the same team_id.""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = await create_scratch_team( + prisma, + team_id=scratch.tag("team"), + admin_user_ids=[world.keys[Actor.PROXY_ADMIN].user_id], + ) + real_key = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="real", + team_id=team_id, + ) + resp = await proxy_client.post( + "/team/key/bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "key_ids": [real_key, real_key, "sk-ghost-" + scratch.prefix], + "update_fields": {"max_budget": 9.0}, + }, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + # Dedupe collapses the two real_key entries → 2 unique tokens. + assert body["total_requested"] == 2 + assert len(body["successful_updates"]) == 1 + assert len(body["failed_updates"]) == 1 + + +# --------------------------------------------------------------------------- +# /key/aliases — pins the handler body lines 5108–5207. PROXY_ADMIN hits +# the broad-scope path; a non-admin caller hits the scoped path through +# _apply_non_admin_alias_scope. +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# /key/info — pins the handler body (lines 3253–3303). PROXY_ADMIN with an +# explicit ghost key 404s; with a real key 200s. Phase 1–3 covered the +# auth matrix; this adds the explicit-key path the matrix doesn't hit. +# --------------------------------------------------------------------------- + + +async def test_key_info_ghost_key_404(proxy_client, scratch, world): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.get( + f"/key/info?key=sk-{scratch.prefix}-ghost", + headers={"Authorization": f"Bearer {seeder}"}, + ) + assert resp.status_code == 404, resp.text + + +async def test_key_info_explicit_existing_key(proxy_client, prisma, scratch, world): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="info-test", + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.get( + f"/key/info?key={cleartext}", + headers={"Authorization": f"Bearer {seeder}"}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["key"] == cleartext + assert "info" in body + # Token hash is stripped from the response (line 3296). + assert "token" not in body["info"] + + +async def test_key_info_no_key_uses_auth_header(proxy_client, world): + """Pin line 3260: `key = key or user_api_key_dict.api_key` — caller's + own key info is returned when no `?key=` is supplied.""" + caller = world.keys[Actor.INTERNAL_USER].cleartext + resp = await proxy_client.get( + "/key/info", + headers={"Authorization": f"Bearer {caller}"}, + ) + assert resp.status_code == 200, resp.text + + +# --------------------------------------------------------------------------- +# /key/generate with budget_limits — pins budget_limits initialization +# inside generate_key_helper_fn (lines 3427–3436). +# --------------------------------------------------------------------------- + + +async def test_key_generate_with_budget_limits(proxy_client, prisma, scratch, world): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "key_alias": scratch.prefix, + "budget_limits": [ + {"max_budget": 5.0, "budget_duration": "1d"}, + {"max_budget": 20.0, "budget_duration": "30d"}, + ], + }, + ) + assert resp.status_code == 200, resp.text + + +async def test_key_aliases_proxy_admin_unscoped(proxy_client, prisma, scratch, world): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + alias = scratch.prefix + "-aliases-test" + await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key_alias": alias}, + ) + resp = await proxy_client.get( + "/key/aliases?size=10", + headers={"Authorization": f"Bearer {seeder}"}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert "aliases" in body + assert "total_count" in body + assert body["current_page"] == 1 + assert body["size"] == 10 + assert ( + alias in body["aliases"] + ), f"newly created alias not in list: {body['aliases']}" + + +async def test_key_aliases_with_search_filter(proxy_client, prisma, scratch, world): + """Pin the `search` ILIKE branch (lines 5166–5168).""" + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + unique_alias = scratch.prefix + "-search-unique-tag" + await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key_alias": unique_alias}, + ) + resp = await proxy_client.get( + f"/key/aliases?search={scratch.prefix}-search", + headers={"Authorization": f"Bearer {seeder}"}, + ) + assert resp.status_code == 200, resp.text + assert unique_alias in resp.json()["aliases"] + + +async def test_key_aliases_non_admin_scoped(proxy_client, world): + """Non-admin caller routes through _apply_non_admin_alias_scope (line + 5161–5164). The exact alias visibility depends on team membership; the + pin is that the call succeeds with a scoped result.""" + caller = world.keys[Actor.INTERNAL_USER].cleartext + resp = await proxy_client.get( + "/key/aliases?size=10", + headers={"Authorization": f"Bearer {caller}"}, + ) + assert resp.status_code == 200, resp.text + assert "aliases" in resp.json() + + +# --------------------------------------------------------------------------- +# /key/list with extended filters — pins _build_filter_conditions branches +# at lines 5280–5388. Each scenario varies one filter so a different +# branch fires. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "query", + [ + "include_created_by_keys=true", + "include_team_keys=true", + "return_full_object=true", + "sort_by=created_at&sort_order=asc", + "key_alias=behavior", + ], + ids=["created_by", "team_keys", "full_object", "sort", "alias_substring"], +) +async def test_key_list_filter_branches(query: str, proxy_client, world): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.get( + f"/key/list?{query}", + headers={"Authorization": f"Bearer {seeder}"}, + ) + assert resp.status_code == 200, resp.text + + +# --------------------------------------------------------------------------- +# /key/regenerate with grace_period — pins _insert_deprecated_key body +# (lines 4168–4202). The old token gets retained in +# LiteLLM_DeprecatedVerificationToken; assert it lands there. +# --------------------------------------------------------------------------- + + +async def test_regenerate_with_grace_period_inserts_deprecated_row( + proxy_client, prisma, scratch, world +): + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="grace", + ) + old_hash = hash_token(cleartext) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/{cleartext}/regenerate", + headers={"Authorization": f"Bearer {seeder}"}, + json={"grace_period": "1h"}, + ) + assert resp.status_code == 200, resp.text + new_key = resp.json()["key"] + # Old token should now be in the deprecated table. + deprecated_row = await prisma.db.litellm_deprecatedverificationtoken.find_unique( + where={"token": old_hash} + ) + assert deprecated_row is not None, "old token not retained in deprecated table" + assert deprecated_row.active_token_id == hash_token(new_key) + assert deprecated_row.revoke_at is not None + # Manual cleanup — scratch prefix sweep doesn't cover this table. + await prisma.db.litellm_deprecatedverificationtoken.delete( + where={"token": old_hash} + ) + + +async def test_regenerate_with_invalid_grace_period_format( + proxy_client, prisma, scratch, world +): + """Invalid grace_period format falls through silently (line 4170–4175); + regenerate still succeeds but no deprecated row is inserted.""" + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.PROXY_ADMIN].user_id, + suffix="grace-bad", + ) + old_hash = hash_token(cleartext) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + f"/key/{cleartext}/regenerate", + headers={"Authorization": f"Bearer {seeder}"}, + json={"grace_period": "totally-not-a-duration"}, + ) + assert resp.status_code == 200, resp.text + deprecated_row = await prisma.db.litellm_deprecatedverificationtoken.find_unique( + where={"token": old_hash} + ) + assert deprecated_row is None, "deprecated row created despite invalid grace_period" + + +async def test_key_list_key_hash_filter_unauthorized( + proxy_client, prisma, scratch, world +): + """validate_key_list_check's key_hash branch (lines 4766–4789): a + cross-tenant non-admin caller asks for a key_hash they don't own → 403. + + `user_belongs_to_keys_team` returns True for any team member, so a + same-team caller is allowed to query peer keys by hash (intentional + per the helper's policy). The 403 path requires a caller who is neither + the key owner, team member, nor admin — i.e. CROSS_ORG_USER. + """ + cleartext = await _seed_token( + prisma, + scratch.prefix, + user_id=world.keys[Actor.OWNER].user_id, + team_id=TEAM_ALPHA, + ) + caller = world.keys[Actor.CROSS_ORG_USER].cleartext + resp = await proxy_client.get( + f"/key/list?key_hash={hash_token(cleartext)}", + headers={"Authorization": f"Bearer {caller}"}, + ) + assert resp.status_code == 403, resp.text diff --git a/tests/proxy_behavior/management/test_key_budget_limits.py b/tests/proxy_behavior/management/test_key_budget_limits.py new file mode 100644 index 000000000000..f5066d60a347 --- /dev/null +++ b/tests/proxy_behavior/management/test_key_budget_limits.py @@ -0,0 +1,383 @@ +"""Phase 4 F1 — payload-level pins for key budget & rate-limit enforcement. + +Pins the five helpers + * _check_key_model_specific_limits (key_management_endpoints.py:931) + * _check_key_rpm_tpm_limits (key_management_endpoints.py:1016) + * _check_team_key_limits (key_management_endpoints.py:1096) + * _check_org_key_limits (key_management_endpoints.py:1284) + * _check_project_key_limits (key_management_endpoints.py:1135) + +Driven through /key/generate. PROXY_ADMIN is the caller so authz never +short-circuits the payload check — Phase 1–3 already pinned authz cleanly. + +`guaranteed_throughput` on either tpm_limit_type or rpm_limit_type is the +trigger that arms `_check_team_key_limits` / `_check_org_key_limits`; without +it both helpers early-return before reading any limit. The project sub-family +covers `_check_project_key_limits`, which has no such gate. + +Each scenario asserts BOTH: + - HTTP status, and + - the DB row state on re-read (created when accepted; absent when rejected). +A response-body check is deliberately avoided per parent plan's anti-snapshot +rule. +""" + +from typing import Any, Dict + +import pytest + +from .actors import Actor +from .conftest import create_scratch_org, create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +# --------------------------------------------------------------------------- +# _check_team_key_limits — aggregate tpm/rpm guard +# --------------------------------------------------------------------------- + +# (id, team_tpm, team_rpm, body_extras, expected_status, detail_substring) +_TEAM_RATE_LIMIT_SCENARIOS = [ + ( + "aggregate/tpm_within_bound", + 1000, + None, + {"tpm_limit": 400, "tpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "aggregate/tpm_over_bound", + 1000, + None, + {"tpm_limit": 2000, "tpm_limit_type": "guaranteed_throughput"}, + 400, + "TPM limit", + ), + ( + "aggregate/rpm_within_bound", + None, + 100, + {"rpm_limit": 40, "rpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "aggregate/rpm_over_bound", + None, + 100, + {"rpm_limit": 250, "rpm_limit_type": "guaranteed_throughput"}, + 400, + "RPM limit", + ), + ( + "aggregate/no_guaranteed_throughput_skips_check", + None, + 10, + # No *_limit_type → helper early-returns even though rpm exceeds team's. + {"rpm_limit": 50}, + 200, + None, + ), +] + + +@pytest.mark.parametrize( + "team_tpm,team_rpm,body_extras,expected_status,detail_substring", + [(a, b, c, d, e) for (_id, a, b, c, d, e) in _TEAM_RATE_LIMIT_SCENARIOS], + ids=[s[0] for s in _TEAM_RATE_LIMIT_SCENARIOS], +) +async def test_check_team_key_limits_aggregate( + team_tpm, + team_rpm, + body_extras: Dict[str, Any], + expected_status: int, + detail_substring, + proxy_client, + prisma, + scratch, + world, +): + team_id = await create_scratch_team( + prisma, + team_id=scratch.tag("team"), + tpm_limit=team_tpm, + rpm_limit=team_rpm, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + body: Dict[str, Any] = { + "key_alias": scratch.prefix, + "team_id": team_id, + **body_extras, + } + resp = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + assert ( + resp.status_code == expected_status + ), f"{body!r} → {resp.status_code}: {resp.text}" + if detail_substring is not None: + assert detail_substring in resp.text, resp.text + + rows = await prisma.db.litellm_verificationtoken.find_many( + where={"key_alias": scratch.prefix} + ) + if expected_status == 200: + assert len(rows) == 1 + else: + assert rows == [], "rejected key leaked a row" + + +# --------------------------------------------------------------------------- +# _check_team_key_limits — model-specific guard (via team metadata) +# --------------------------------------------------------------------------- + +# Body shape: {"model_rpm_limit": {"gpt-4": }, "rpm_limit_type": "guaranteed_throughput"} +# Team metadata supplies the matching per-model cap. +_TEAM_MODEL_LIMIT_SCENARIOS = [ + ( + "model_rpm/within_bound", + {"model_rpm_limit": {"gpt-4": 30}}, + {"model_rpm_limit": {"gpt-4": 20}, "rpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "model_rpm/over_bound", + {"model_rpm_limit": {"gpt-4": 30}}, + {"model_rpm_limit": {"gpt-4": 100}, "rpm_limit_type": "guaranteed_throughput"}, + 400, + "RPM", + ), + ( + "model_tpm/within_bound", + {"model_tpm_limit": {"gpt-4": 500}}, + {"model_tpm_limit": {"gpt-4": 200}, "tpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "model_tpm/over_bound", + {"model_tpm_limit": {"gpt-4": 500}}, + {"model_tpm_limit": {"gpt-4": 5000}, "tpm_limit_type": "guaranteed_throughput"}, + 400, + "TPM", + ), +] + + +@pytest.mark.parametrize( + "team_metadata,body_extras,expected_status,detail_substring", + [(b, c, d, e) for (_id, b, c, d, e) in _TEAM_MODEL_LIMIT_SCENARIOS], + ids=[s[0] for s in _TEAM_MODEL_LIMIT_SCENARIOS], +) +async def test_check_team_key_limits_model_specific( + team_metadata, + body_extras: Dict[str, Any], + expected_status: int, + detail_substring, + proxy_client, + prisma, + scratch, + world, +): + team_id = await create_scratch_team( + prisma, + team_id=scratch.tag("team"), + metadata=team_metadata, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + body: Dict[str, Any] = { + "key_alias": scratch.prefix, + "team_id": team_id, + **body_extras, + } + resp = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + assert ( + resp.status_code == expected_status + ), f"{body!r} → {resp.status_code}: {resp.text}" + if detail_substring is not None: + assert detail_substring in resp.text, resp.text + rows = await prisma.db.litellm_verificationtoken.find_many( + where={"key_alias": scratch.prefix} + ) + assert len(rows) == (1 if expected_status == 200 else 0) + + +# --------------------------------------------------------------------------- +# _check_org_key_limits — aggregate tpm/rpm guard via budget table +# +# Behavior pin, not a regression target: /key/generate (line 889) and +# /key/update (line 2310) both call `get_org_object` WITHOUT +# `include_budget_table=True`, so `org_table.litellm_budget_table` is None +# at guard time and the aggregate path silently no-ops. Over-bound payloads +# therefore land as 200, not 400. Documenting that here so a future change +# that flips include_budget_table=True or moves the guard pre-load would +# turn these into reds — exactly the regression-tripwire shape Phase 4 wants. +# The model-specific guard below DOES fire because it reads org metadata, +# which is loaded directly on the org row (no relation include needed). +# --------------------------------------------------------------------------- + +_ORG_RATE_LIMIT_SCENARIOS = [ + ( + "org/tpm_within_bound", + {"max_budget": None, "tpm_limit": 1000, "rpm_limit": None}, + {"tpm_limit": 400, "tpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "org/tpm_over_bound_unenforced_no_include_budget_table", + {"max_budget": None, "tpm_limit": 1000, "rpm_limit": None}, + {"tpm_limit": 2000, "tpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "org/rpm_within_bound", + {"max_budget": None, "tpm_limit": None, "rpm_limit": 100}, + {"rpm_limit": 40, "rpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "org/rpm_over_bound_unenforced_no_include_budget_table", + {"max_budget": None, "tpm_limit": None, "rpm_limit": 100}, + {"rpm_limit": 250, "rpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "org/no_guaranteed_throughput_skips_check", + {"max_budget": None, "tpm_limit": None, "rpm_limit": 10}, + {"rpm_limit": 50}, + 200, + None, + ), +] + + +@pytest.mark.parametrize( + "org_budget,body_extras,expected_status,detail_substring", + [(b, c, d, e) for (_id, b, c, d, e) in _ORG_RATE_LIMIT_SCENARIOS], + ids=[s[0] for s in _ORG_RATE_LIMIT_SCENARIOS], +) +async def test_check_org_key_limits_aggregate( + org_budget: Dict[str, Any], + body_extras: Dict[str, Any], + expected_status: int, + detail_substring, + proxy_client, + prisma, + scratch, + world, +): + org_id = await create_scratch_org(prisma, scratch.prefix, **org_budget) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + body: Dict[str, Any] = { + "key_alias": scratch.prefix, + "organization_id": org_id, + **body_extras, + } + resp = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + assert ( + resp.status_code == expected_status + ), f"{body!r} → {resp.status_code}: {resp.text}" + if detail_substring is not None: + assert detail_substring in resp.text, resp.text + rows = await prisma.db.litellm_verificationtoken.find_many( + where={"key_alias": scratch.prefix} + ) + assert len(rows) == (1 if expected_status == 200 else 0) + + +# --------------------------------------------------------------------------- +# _check_org_key_limits — model-specific guard via org metadata +# --------------------------------------------------------------------------- + +_ORG_MODEL_LIMIT_SCENARIOS = [ + ( + "org_model_rpm/within_bound", + {"model_rpm_limit": {"gpt-4": 30}}, + {"model_rpm_limit": {"gpt-4": 20}, "rpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "org_model_rpm/over_bound", + {"model_rpm_limit": {"gpt-4": 30}}, + {"model_rpm_limit": {"gpt-4": 100}, "rpm_limit_type": "guaranteed_throughput"}, + 400, + "RPM", + ), + ( + "org_model_tpm/within_bound", + {"model_tpm_limit": {"gpt-4": 500}}, + {"model_tpm_limit": {"gpt-4": 200}, "tpm_limit_type": "guaranteed_throughput"}, + 200, + None, + ), + ( + "org_model_tpm/over_bound", + {"model_tpm_limit": {"gpt-4": 500}}, + {"model_tpm_limit": {"gpt-4": 5000}, "tpm_limit_type": "guaranteed_throughput"}, + 400, + "TPM", + ), +] + + +@pytest.mark.parametrize( + "org_metadata,body_extras,expected_status,detail_substring", + [(b, c, d, e) for (_id, b, c, d, e) in _ORG_MODEL_LIMIT_SCENARIOS], + ids=[s[0] for s in _ORG_MODEL_LIMIT_SCENARIOS], +) +async def test_check_org_key_limits_model_specific( + org_metadata, + body_extras: Dict[str, Any], + expected_status: int, + detail_substring, + proxy_client, + prisma, + scratch, + world, +): + org_id = await create_scratch_org(prisma, scratch.prefix, metadata=org_metadata) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + body: Dict[str, Any] = { + "key_alias": scratch.prefix, + "organization_id": org_id, + **body_extras, + } + resp = await proxy_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + assert ( + resp.status_code == expected_status + ), f"{body!r} → {resp.status_code}: {resp.text}" + if detail_substring is not None: + assert detail_substring in resp.text, resp.text + rows = await prisma.db.litellm_verificationtoken.find_many( + where={"key_alias": scratch.prefix} + ) + assert len(rows) == (1 if expected_status == 200 else 0) + + +# --------------------------------------------------------------------------- +# Project sub-family deferral — see plan §4-F1 ("project surface proves thin"): +# `get_project_object` requires an LiteLLM_ProjectTable seeder + cache wiring +# that the harness does not yet ship; pinning it would force a wider +# conftest change for a single-helper close-out. Tracked as "deferred" in +# the PR4.M3 follow-up box. diff --git a/tests/proxy_behavior/management/test_key_team_change.py b/tests/proxy_behavior/management/test_key_team_change.py new file mode 100644 index 000000000000..3bd4a0af0e2f --- /dev/null +++ b/tests/proxy_behavior/management/test_key_team_change.py @@ -0,0 +1,229 @@ +"""Phase 4 F2 — payload-level pins for key↔team reassignment. + +Pins `validate_key_team_change` (key_management_endpoints.py:2953), reached +via /key/update when the request changes `team_id`. + +The handler runs four guards in order; each scenario isolates one path and +asserts the rejected row's `team_id` is UNCHANGED on a fresh DB re-read — +the regression shape that matters for cross-team / IDOR-class bugs is +"row mutated despite the helper raising", and the only way to catch it +is to compare the persisted state, not the response body. + +The accepted scenario pins the happy path: re-read confirms team_id +flipped to the new team and the row is otherwise intact. +""" + +import uuid +from typing import Any, Dict, Optional + +import pytest +from prisma import Json + +from litellm.proxy.utils import hash_token + +from .actors import TEAM_ALPHA, Actor +from .conftest import create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + +KEY_MODEL = "phase4-f2-key-model" + + +async def _seed_key_with_limits( + prisma, + scratch_prefix: str, + *, + user_id: str, + team_id: str, + models: Optional[list] = None, + tpm_limit: Optional[int] = None, + rpm_limit: Optional[int] = None, +) -> str: + """Raw-seed a scratch key with explicit models / tpm / rpm — /key/generate + can't set these against a non-throughput team without firing F1's guards. + Returns the cleartext key for /key/update calls.""" + cleartext = "sk-" + uuid.uuid4().hex + data: Dict[str, Any] = { + "token": hash_token(cleartext), + "key_alias": f"{scratch_prefix}-key", + "key_name": f"{scratch_prefix}-key", + "user_id": user_id, + "team_id": team_id, + "models": models or [], + } + if tpm_limit is not None: + data["tpm_limit"] = tpm_limit + if rpm_limit is not None: + data["rpm_limit"] = rpm_limit + await prisma.db.litellm_verificationtoken.create(data=data) + return cleartext + + +# --------------------------------------------------------------------------- +# Accepted — proxy admin moves a key into a scratch team that has the model, +# accommodates the limits, and has the key owner as a member. +# --------------------------------------------------------------------------- + + +async def test_key_team_change_accepted(proxy_client, prisma, scratch, world): + owner_id = world.keys[Actor.OWNER].user_id + target_team = await create_scratch_team( + prisma, + team_id=scratch.tag("target"), + member_user_ids=[owner_id], + models=[KEY_MODEL], + tpm_limit=10_000, + rpm_limit=1_000, + ) + key_cleartext = await _seed_key_with_limits( + prisma, + scratch.prefix, + user_id=owner_id, + team_id=TEAM_ALPHA, + models=[KEY_MODEL], + tpm_limit=500, + rpm_limit=50, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + + resp = await proxy_client.post( + "/key/update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key": key_cleartext, "team_id": target_team}, + ) + assert resp.status_code == 200, resp.text + + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(key_cleartext)} + ) + assert row is not None + assert row.team_id == target_team, "key did not move to target team" + + +# --------------------------------------------------------------------------- +# Rejected — each path verifies row.team_id is UNCHANGED on DB re-read. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "scenario", + [ + "team_lacks_model", + "key_tpm_exceeds_team_tpm", + "key_rpm_exceeds_team_rpm", + "key_owner_not_team_member", + ], +) +async def test_key_team_change_rejected_guards( + scenario: str, proxy_client, prisma, scratch, world +): + owner_id = world.keys[Actor.OWNER].user_id + + team_kwargs: Dict[str, Any] = { + "member_user_ids": [owner_id], + "models": [KEY_MODEL], + "tpm_limit": 10_000, + "rpm_limit": 1_000, + } + key_kwargs: Dict[str, Any] = { + "models": [KEY_MODEL], + "tpm_limit": 500, + "rpm_limit": 50, + } + + if scenario == "team_lacks_model": + team_kwargs["models"] = ["something-else"] + elif scenario == "key_tpm_exceeds_team_tpm": + team_kwargs["tpm_limit"] = 100 # < 500 + elif scenario == "key_rpm_exceeds_team_rpm": + team_kwargs["rpm_limit"] = 5 # < 50 + elif scenario == "key_owner_not_team_member": + team_kwargs["member_user_ids"] = [] + + target_team = await create_scratch_team( + prisma, + team_id=scratch.tag("target"), + **team_kwargs, + ) + key_cleartext = await _seed_key_with_limits( + prisma, + scratch.prefix, + user_id=owner_id, + team_id=TEAM_ALPHA, + **key_kwargs, + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + + resp = await proxy_client.post( + "/key/update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"key": key_cleartext, "team_id": target_team}, + ) + # The model-mismatch path lands as 400 (ProxyException from + # _can_object_call_model); the limit + membership paths land as 403 + # (validate_key_team_change's own HTTPException). Both shapes count as + # "rejected" for the pin; what matters is the row stayed put. + assert resp.status_code in ( + 400, + 403, + ), f"{scenario}: expected 400/403, got {resp.status_code}: {resp.text}" + + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(key_cleartext)} + ) + assert row is not None + assert row.team_id == TEAM_ALPHA, ( + f"{scenario}: row team_id mutated despite rejection — " f"got {row.team_id!r}" + ) + + +# --------------------------------------------------------------------------- +# Rejected — initiator is neither proxy admin, team admin, nor permission- +# granted. Pinned separately because the source team here must also have the +# initiator listed as a non-admin member (otherwise the earlier user_id +# membership guard fires first). +# --------------------------------------------------------------------------- + + +async def test_key_team_change_rejected_initiator_not_admin( + proxy_client, prisma, scratch, world +): + owner_id = world.keys[Actor.OWNER].user_id + target_team = await create_scratch_team( + prisma, + team_id=scratch.tag("target"), + # Owner is the key's user_id; member of the team to clear membership + # guard. Internal-user role on world.keys[INTERNAL_USER] is also a + # member of TEAM_ALPHA but never an admin → role-gate path fires. + member_user_ids=[owner_id, world.keys[Actor.INTERNAL_USER].user_id], + models=[KEY_MODEL], + tpm_limit=10_000, + rpm_limit=1_000, + ) + key_cleartext = await _seed_key_with_limits( + prisma, + scratch.prefix, + user_id=owner_id, + team_id=TEAM_ALPHA, + models=[KEY_MODEL], + tpm_limit=500, + rpm_limit=50, + ) + # The internal_user actor is the initiator: a TEAM_ALPHA member, but not + # an admin anywhere, and has no team_member_permissions for /key/update. + initiator = world.keys[Actor.INTERNAL_USER].cleartext + + resp = await proxy_client.post( + "/key/update", + headers={"Authorization": f"Bearer {initiator}"}, + json={"key": key_cleartext, "team_id": target_team}, + ) + assert resp.status_code in ( + 401, + 403, + ), f"expected 401/403, got {resp.status_code}: {resp.text}" + row = await prisma.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(key_cleartext)} + ) + assert row is not None + assert row.team_id == TEAM_ALPHA, "row team_id mutated despite rejection" diff --git a/tests/proxy_behavior/management/test_team_budget_limits.py b/tests/proxy_behavior/management/test_team_budget_limits.py new file mode 100644 index 000000000000..1534cee2b2eb --- /dev/null +++ b/tests/proxy_behavior/management/test_team_budget_limits.py @@ -0,0 +1,321 @@ +"""Phase 4 F3 — payload-level pins for team budget & rate-limit enforcement. + +Pins the five helpers + * _check_team_model_specific_limits (team_endpoints.py:442) + * _check_team_rpm_tpm_limits (team_endpoints.py:527) + * check_org_team_model_specific_limits (team_endpoints.py:569) + * check_org_team_rpm_tpm_limits (team_endpoints.py:603) + * _check_org_team_limits (team_endpoints.py:628) + * _check_user_team_limits (team_endpoints.py:734) + +Driven through /team/new + /team/update. + +Structural finding pinned here, identical in shape to F1's org aggregate: +both call sites (lines 985 + 1751) load the org via `get_org_object` +WITHOUT `include_budget_table=True`, so `org_table.litellm_budget_table` +is `None` and the org max_budget / org tpm / org rpm guards inside +`_check_org_team_limits` (lines 641–694, 670–694) silently no-op. The +`models` subset guard (lines 654–667) IS reachable because it reads +`org_table.models` directly. The `_check_user_team_limits` guards reach +all branches through `user_api_key_dict`, no relation include needed. +""" + +import uuid +from typing import Any, Dict, Optional + +import pytest + +from litellm.proxy.utils import hash_token + +from .actors import Actor +from .conftest import create_scratch_org, create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +async def _seed_scratch_actor_with_caps( + prisma, + scratch_prefix: str, + *, + user_role: str = "internal_user", + models: Optional[list] = None, + max_budget: Optional[float] = None, + tpm_limit: Optional[int] = None, + rpm_limit: Optional[int] = None, +) -> str: + """Raw-seed a scratch actor + verification token, with the token carrying + explicit caps that flow into `user_api_key_dict` at request time. + + Returns cleartext key. Used by F3 user-team-limit scenarios where the + caller's caps drive the rejection. Uses 'internal_user' role plus a + token-level `allowed_routes` whitelist of /team/new + /team/update — + that whitelist is the only way past the admin-only role gate in + `RouteChecks.non_proxy_admin_allowed_routes_check`; without it the + non-admin caller would 401 before `_check_user_team_limits` ever fires. + """ + user_id = f"{scratch_prefix}-team-creator" + cleartext = "sk-" + uuid.uuid4().hex + await prisma.db.litellm_usertable.create( + data={ + "user_id": user_id, + "user_role": user_role, + "max_budget": max_budget, + } + ) + token_data: Dict[str, Any] = { + "token": hash_token(cleartext), + "key_name": f"{scratch_prefix}-team-creator-key", + "key_alias": f"{scratch_prefix}-team-creator-alias", + "user_id": user_id, + "models": models if models is not None else [], + "allowed_routes": ["/team/new", "/team/update"], + } + if tpm_limit is not None: + token_data["tpm_limit"] = tpm_limit + if rpm_limit is not None: + token_data["rpm_limit"] = rpm_limit + await prisma.db.litellm_verificationtoken.create(data=token_data) + return cleartext + + +# --------------------------------------------------------------------------- +# _check_org_team_limits — models subset guard (the one path that reaches) +# --------------------------------------------------------------------------- + +_ORG_MODEL_SCENARIOS = [ + ( + "org_models/team_subset_accepted", + ["allowed-model"], + {"models": ["allowed-model"]}, + 200, + ), + ( + "org_models/team_extra_model_rejected", + ["allowed-model"], + {"models": ["forbidden-model"]}, + 400, + ), + ( + "org_models/all_proxy_models_skips_check", + ["all-proxy-models"], + {"models": ["any-model-at-all"]}, + 200, + ), +] + + +@pytest.mark.parametrize( + "org_models,body_extras,expected_status", + [(b, c, d) for (_id, b, c, d) in _ORG_MODEL_SCENARIOS], + ids=[s[0] for s in _ORG_MODEL_SCENARIOS], +) +async def test_check_org_team_limits_models_subset( + org_models, + body_extras: Dict[str, Any], + expected_status: int, + proxy_client, + prisma, + scratch, + world, +): + org_id = await create_scratch_org(prisma, scratch.prefix, models=org_models) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = scratch.tag("team") + body: Dict[str, Any] = { + "team_id": team_id, + "team_alias": scratch.prefix, + "organization_id": org_id, + **body_extras, + } + resp = await proxy_client.post( + "/team/new", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + assert ( + resp.status_code == expected_status + ), f"{body!r} → {resp.status_code}: {resp.text}" + + rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + assert len(rows) == (1 if expected_status == 200 else 0) + + +# --------------------------------------------------------------------------- +# _check_org_team_limits — budget / tpm / rpm structurally unreachable +# (org_table.litellm_budget_table is None at guard time). Pin the +# no-op behavior so a future change that flips include_budget_table=True +# turns these into reds. +# --------------------------------------------------------------------------- + +_ORG_BUDGET_DEAD_SCENARIOS = [ + ( + "org_budget/over_max_budget_unenforced", + {"max_budget": 100, "tpm_limit": None, "rpm_limit": None}, + {"max_budget": 999_999}, + ), + ( + "org_tpm/over_unenforced", + {"max_budget": None, "tpm_limit": 100, "rpm_limit": None}, + {"tpm_limit": 999_999}, + ), + ( + "org_rpm/over_unenforced", + {"max_budget": None, "tpm_limit": None, "rpm_limit": 100}, + {"rpm_limit": 999_999}, + ), +] + + +@pytest.mark.parametrize( + "org_budget,body_extras", + [(b, c) for (_id, b, c) in _ORG_BUDGET_DEAD_SCENARIOS], + ids=[s[0] for s in _ORG_BUDGET_DEAD_SCENARIOS], +) +async def test_check_org_team_limits_budget_dead_code_pin( + org_budget, + body_extras: Dict[str, Any], + proxy_client, + prisma, + scratch, + world, +): + org_id = await create_scratch_org(prisma, scratch.prefix, **org_budget) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + team_id = scratch.tag("team") + resp = await proxy_client.post( + "/team/new", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "team_alias": scratch.prefix, + "organization_id": org_id, + **body_extras, + }, + ) + assert resp.status_code == 200, resp.text + rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + assert len(rows) == 1 + + +# --------------------------------------------------------------------------- +# _check_user_team_limits — fires for standalone (no-org) teams created by +# a non-admin caller. Each guard reads from user_api_key_dict / user_obj. +# --------------------------------------------------------------------------- + +_USER_LIMIT_SCENARIOS = [ + ( + "user_max_budget/within", + {"max_budget": 100.0, "models": [], "tpm_limit": None, "rpm_limit": None}, + {"max_budget": 50.0}, + 200, + ), + ( + "user_max_budget/over", + {"max_budget": 100.0, "models": [], "tpm_limit": None, "rpm_limit": None}, + {"max_budget": 1000.0}, + 400, + ), + ( + "user_models/subset", + {"max_budget": None, "models": ["m-a"], "tpm_limit": None, "rpm_limit": None}, + {"models": ["m-a"]}, + 200, + ), + ( + "user_models/superset_rejected", + {"max_budget": None, "models": ["m-a"], "tpm_limit": None, "rpm_limit": None}, + {"models": ["m-a", "m-b"]}, + 400, + ), + ( + "user_tpm/within", + {"max_budget": None, "models": [], "tpm_limit": 1000, "rpm_limit": None}, + {"tpm_limit": 500}, + 200, + ), + ( + "user_tpm/over", + {"max_budget": None, "models": [], "tpm_limit": 1000, "rpm_limit": None}, + {"tpm_limit": 2000}, + 400, + ), + ( + "user_rpm/within", + {"max_budget": None, "models": [], "tpm_limit": None, "rpm_limit": 100}, + {"rpm_limit": 50}, + 200, + ), + ( + "user_rpm/over", + {"max_budget": None, "models": [], "tpm_limit": None, "rpm_limit": 100}, + {"rpm_limit": 250}, + 400, + ), +] + + +@pytest.mark.parametrize( + "actor_caps,body_extras,expected_status", + [(b, c, d) for (_id, b, c, d) in _USER_LIMIT_SCENARIOS], + ids=[s[0] for s in _USER_LIMIT_SCENARIOS], +) +async def test_check_user_team_limits( + actor_caps, + body_extras: Dict[str, Any], + expected_status: int, + proxy_client, + prisma, + scratch, +): + caller = await _seed_scratch_actor_with_caps(prisma, scratch.prefix, **actor_caps) + team_id = scratch.tag("team") + resp = await proxy_client.post( + "/team/new", + headers={"Authorization": f"Bearer {caller}"}, + json={ + "team_id": team_id, + "team_alias": scratch.prefix, + # Standalone team — no organization_id, so user-limit guard fires. + **body_extras, + }, + ) + assert ( + resp.status_code == expected_status + ), f"caps={actor_caps} body={body_extras} → {resp.status_code}: {resp.text}" + + rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + assert len(rows) == (1 if expected_status == 200 else 0) + + +# --------------------------------------------------------------------------- +# /team/update path — _check_user_team_limits on existing team, no-org. +# Pin one over-budget rejection here so the update-side wiring is also +# covered (the update path is a second call site with its own data shape). +# --------------------------------------------------------------------------- + + +async def test_team_update_user_limit_rejected(proxy_client, prisma, scratch): + caller_cleartext = await _seed_scratch_actor_with_caps( + prisma, + scratch.prefix, + max_budget=100.0, + ) + creator_user_id = f"{scratch.prefix}-team-creator" + # Team must exist before /team/update; seed a standalone scratch team + # owned by the same actor so the update authz gate passes. + team_id = await create_scratch_team( + prisma, + team_id=scratch.tag("team"), + admin_user_ids=[creator_user_id], + max_budget=50.0, + ) + resp = await proxy_client.post( + "/team/update", + headers={"Authorization": f"Bearer {caller_cleartext}"}, + json={"team_id": team_id, "max_budget": 999.0}, + ) + assert resp.status_code == 400, resp.text + + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + assert row is not None + assert row.max_budget == 50.0, "row max_budget mutated despite rejection" diff --git a/tests/proxy_behavior/management/test_team_info.py b/tests/proxy_behavior/management/test_team_info.py index 518099421134..ad019207c82d 100644 --- a/tests/proxy_behavior/management/test_team_info.py +++ b/tests/proxy_behavior/management/test_team_info.py @@ -68,3 +68,19 @@ async def test_team_info_authz_matrix( body = resp.json() assert body["team_id"] == target_team_id assert body["team_info"]["team_id"] == target_team_id + + +# Phase 4 F6 — explicit pin on the `_verify_team_access` 403 message string. +# alpha/org_b_admin already covers the branch in the matrix; this guard +# turns a silent rename of the exception detail into a CI red, which is the +# behavior tripwire that the matrix's status-only assertion cannot catch. +async def test_team_info_org_admin_cross_org_rejection_detail(proxy_client, world): + resp = await proxy_client.get( + f"/team/info?team_id={world.team_alpha_id}", + headers={"Authorization": f"Bearer {world.keys[Actor.ORG_B_ADMIN].cleartext}"}, + ) + assert resp.status_code == 403, resp.text + # validate_membership() at GET /team/info raises with this exact phrase. + # Pinning it lock-step locks down the visible auth message — a rename + # would flip CI red even when the status stays 403. + assert "not authorized to access this team" in resp.text, resp.text diff --git a/tests/proxy_behavior/management/test_team_member_info_validation.py b/tests/proxy_behavior/management/test_team_member_info_validation.py new file mode 100644 index 000000000000..5139865d2b42 --- /dev/null +++ b/tests/proxy_behavior/management/test_team_member_info_validation.py @@ -0,0 +1,189 @@ +"""Phase 4 F4 — payload-level pins for member-info population. + +Pins `_validate_and_populate_member_user_info` (team_endpoints.py:2275), +reached via /team/member_add. + +PROXY_ADMIN is the caller so the upstream `_validate_team_member_add_permissions` +gate never short-circuits the payload check. Each scenario asserts BOTH the +HTTP status and the DB end-state — for accepted cases, the +LiteLLM_TeamMembership row reflects the resolved user_id (the regression +shape: a payload that silently lands the membership against the WRONG +user_id is invisible from response-body alone). +""" + +from typing import Any, Dict, Optional + +import pytest + +from .actors import Actor +from .conftest import create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +async def _seed_scratch_user( + prisma, + scratch_prefix: str, + *, + suffix: str, + user_email: Optional[str] = None, +) -> str: + """Raw-seed a scratch-prefixed user row; returns user_id. Scratch teardown + reclaims by user_id prefix.""" + user_id = f"{scratch_prefix}-{suffix}" + data: Dict[str, Any] = {"user_id": user_id, "user_role": "internal_user"} + if user_email is not None: + data["user_email"] = user_email + await prisma.db.litellm_usertable.create(data=data) + return user_id + + +# --------------------------------------------------------------------------- +# Both None → 400 ("Either user_id or user_email must be provided") +# --------------------------------------------------------------------------- + + +async def test_member_add_both_none_rejected(proxy_client, prisma, scratch, world): + team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/member_add", + headers={"Authorization": f"Bearer {seeder}"}, + json={"team_id": team_id, "member": {"role": "user"}}, + ) + # The Pydantic Member model may also catch this at the validation layer + # (422). Either shape proves the empty-Member payload is rejected before + # any membership row is written — pin both. + assert resp.status_code in (400, 422), resp.text + rows = await prisma.db.litellm_teammembership.find_many(where={"team_id": team_id}) + assert rows == [], "empty-Member payload leaked a membership row" + + +# --------------------------------------------------------------------------- +# Email + id given, but they point at different users → 400 +# --------------------------------------------------------------------------- + + +async def test_member_add_email_id_mismatch_rejected( + proxy_client, prisma, scratch, world +): + email = f"{scratch.prefix}-mismatch@example.com" + real_user_id = await _seed_scratch_user( + prisma, scratch.prefix, suffix="real", user_email=email + ) + other_user_id = await _seed_scratch_user(prisma, scratch.prefix, suffix="other") + assert real_user_id != other_user_id # sanity + team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/member_add", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "member": { + "role": "user", + "user_email": email, + "user_id": other_user_id, + }, + }, + ) + assert resp.status_code == 400, resp.text + assert "do not belong to the same user" in resp.text, resp.text + rows = await prisma.db.litellm_teammembership.find_many(where={"team_id": team_id}) + assert rows == [], "mismatch payload leaked a membership row" + + +# --------------------------------------------------------------------------- +# Email-only resolves to user_id when exactly one user matches +# --------------------------------------------------------------------------- + + +async def test_member_add_email_only_resolves_user_id( + proxy_client, prisma, scratch, world +): + email = f"{scratch.prefix}-resolve@example.com" + user_id = await _seed_scratch_user( + prisma, scratch.prefix, suffix="lookup", user_email=email + ) + team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/member_add", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "member": {"role": "user", "user_email": email}, + }, + ) + assert resp.status_code == 200, resp.text + # litellm_teammembership rows are only written when a per-member budget + # is assigned; the default member-add path stores membership in the + # team's members_with_roles JSON. Re-read that and assert the resolved + # user_id landed — the regression shape is "email resolved to the WRONG + # user_id and was silently written to members_with_roles". + team_row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + assert team_row is not None + member_user_ids = [m.get("user_id") for m in team_row.members_with_roles] + assert ( + user_id in member_user_ids + ), f"email did not resolve to {user_id}; members={member_user_ids}" + + +# --------------------------------------------------------------------------- +# id-only, user does NOT yet exist — passes through, member is upserted. +# --------------------------------------------------------------------------- + + +async def test_member_add_unknown_user_id_upserted( + proxy_client, prisma, scratch, world +): + team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) + new_user_id = f"{scratch.prefix}-fresh" + # Sanity — user does not exist yet. + pre = await prisma.db.litellm_usertable.find_unique(where={"user_id": new_user_id}) + assert pre is None + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/member_add", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "member": {"role": "user", "user_id": new_user_id}, + }, + ) + assert resp.status_code == 200, resp.text + post = await prisma.db.litellm_usertable.find_unique(where={"user_id": new_user_id}) + assert post is not None, "user_id was not upserted" + # The user row was created with NULL email (the helper returned the + # member as-is, no email lookup happened because the user didn't exist). + assert ( + post.user_email is None + ), f"upserted user has unexpected email: {post.user_email!r}" + + +# --------------------------------------------------------------------------- +# Duplicate-email rejection — two scratch users share an email; email-only +# add → 400 with "Multiple users found" detail. +# --------------------------------------------------------------------------- + + +async def test_member_add_duplicate_email_rejected( + proxy_client, prisma, scratch, world +): + email = f"{scratch.prefix}-dup@example.com" + await _seed_scratch_user(prisma, scratch.prefix, suffix="dup1", user_email=email) + await _seed_scratch_user(prisma, scratch.prefix, suffix="dup2", user_email=email) + team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/member_add", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_id": team_id, + "member": {"role": "user", "user_email": email}, + }, + ) + assert resp.status_code == 400, resp.text + assert "Multiple users found" in resp.text, resp.text + rows = await prisma.db.litellm_teammembership.find_many(where={"team_id": team_id}) + assert rows == [], "duplicate-email payload leaked a membership row" diff --git a/tests/proxy_behavior/management/test_team_permissions_bulk_update.py b/tests/proxy_behavior/management/test_team_permissions_bulk_update.py new file mode 100644 index 000000000000..552c3526eea4 --- /dev/null +++ b/tests/proxy_behavior/management/test_team_permissions_bulk_update.py @@ -0,0 +1,245 @@ +"""Phase 4 F5 — payload-level pins for /team/permissions_bulk_update. + +Pins the three helpers + * _compute_and_batch_updates (team_endpoints.py:4887) + * _append_permissions_to_specific_teams (team_endpoints.py:4913) + * _append_permissions_to_all_teams (team_endpoints.py:4932) + +The route is admin-only — PROXY_ADMIN is the only legal caller. The +contracts under test: + * specific-team list → ONLY listed teams mutate; every other team + (including world teams) is byte-identical on re-read. + * apply_to_all_teams → every team gains the permission, idempotently + merged (re-running with the same permission is a no-op). + * unknown team_id → 404 (the missing_ids guard); no mutation anywhere. + * malformed payload (neither / both selector flags) → 400; no mutation. + +The all-teams scenario mutates world teams as a deliberate side effect of +the path under test. The test snapshots every team's +`team_member_permissions` up front and restores them on exit so the +read-world stays immutable for downstream tests. +""" + +import pytest + +from .actors import TEAM_ALPHA, TEAM_BETA, TEAM_GAMMA, Actor +from .conftest import create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +# --------------------------------------------------------------------------- +# Specific-team list — only the listed team mutates. +# --------------------------------------------------------------------------- + + +async def test_bulk_update_specific_team_only_mutates_listed( + proxy_client, prisma, scratch, world +): + target = await create_scratch_team( + prisma, + team_id=scratch.tag("target"), + team_member_permissions=[], + ) + bystander = await create_scratch_team( + prisma, + team_id=scratch.tag("bystander"), + team_member_permissions=["pre-existing-perm"], + ) + # Snapshot world teams so we can assert byte-equality on re-read. + world_team_ids = [TEAM_ALPHA, TEAM_BETA, TEAM_GAMMA] + before = {} + for tid in world_team_ids + [bystander]: + row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": tid}) + before[tid] = list(row.team_member_permissions or []) + + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + perm = "/key/info" + resp = await proxy_client.post( + "/team/permissions_bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"team_ids": [target], "permissions": [perm]}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["teams_updated"] == 1 + + target_row = await prisma.db.litellm_teamtable.find_unique( + where={"team_id": target} + ) + assert perm in ( + target_row.team_member_permissions or [] + ), f"target team did not gain perm; got={target_row.team_member_permissions}" + + for tid, before_perms in before.items(): + after_row = await prisma.db.litellm_teamtable.find_unique( + where={"team_id": tid} + ) + assert list(after_row.team_member_permissions or []) == before_perms, ( + f"untouched team {tid} mutated: " + f"{before_perms} → {list(after_row.team_member_permissions or [])}" + ) + + +# --------------------------------------------------------------------------- +# Specific-team — repeated call with same permission is a no-op (the +# `permissions_to_add <= existing` short-circuit in _compute_and_batch_updates). +# --------------------------------------------------------------------------- + + +async def test_bulk_update_specific_team_idempotent( + proxy_client, prisma, scratch, world +): + target = await create_scratch_team( + prisma, + team_id=scratch.tag("target"), + team_member_permissions=["/key/info"], + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/permissions_bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_ids": [target], + "permissions": ["/key/info"], # already present + }, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["teams_updated"] == 0, "idempotent add should report 0" + + +# --------------------------------------------------------------------------- +# Unknown team_id — 404, no mutation anywhere. +# --------------------------------------------------------------------------- + + +async def test_bulk_update_unknown_team_id_rejected( + proxy_client, prisma, scratch, world +): + existing = await create_scratch_team( + prisma, + team_id=scratch.tag("real"), + team_member_permissions=[], + ) + before = list( + ( + await prisma.db.litellm_teamtable.find_unique(where={"team_id": existing}) + ).team_member_permissions + or [] + ) + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/permissions_bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={ + "team_ids": [existing, f"{scratch.prefix}-ghost"], + "permissions": ["/key/info"], + }, + ) + # The missing_ids check raises 404, but the global exception handler + # may wrap it. Either 404 or 400 with "not found" detail counts. + assert resp.status_code in (400, 404), resp.text + assert "not found" in resp.text.lower() or "ghost" in resp.text, resp.text + # Critical: the partial-success regression shape is "real team got + # mutated before the ghost-id check ran". Re-read and assert it didn't. + after_real = await prisma.db.litellm_teamtable.find_unique( + where={"team_id": existing} + ) + assert ( + list(after_real.team_member_permissions or []) == before + ), "partial mutation: real team changed despite ghost-id rejection" + + +# --------------------------------------------------------------------------- +# Selector validation — neither flag → 400; both flags → 400. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "body,expected_substring", + [ + ( + {"permissions": ["/key/info"]}, + "team_ids or set apply_to_all_teams", + ), + ( + { + "permissions": ["/key/info"], + "team_ids": ["t"], + "apply_to_all_teams": True, + }, + "Cannot set both", + ), + ], + ids=["neither_selector", "both_selectors"], +) +async def test_bulk_update_selector_validation( + body, expected_substring: str, proxy_client, world +): + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/permissions_bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json=body, + ) + assert resp.status_code == 400, resp.text + assert expected_substring in resp.text, resp.text + + +# --------------------------------------------------------------------------- +# apply_to_all_teams — mutates every team. Snapshot + restore world teams +# so the read-world contract holds for downstream tests. +# --------------------------------------------------------------------------- + + +async def test_bulk_update_apply_to_all_mutates_every_team( + proxy_client, prisma, scratch, world +): + # Two scratch teams so we can assert "every" includes our own targets. + a = await create_scratch_team( + prisma, team_id=scratch.tag("a"), team_member_permissions=[] + ) + b = await create_scratch_team( + prisma, team_id=scratch.tag("b"), team_member_permissions=["other"] + ) + perm = "/key/health" # distinct from the specific-team scenarios above + + # Snapshot every team's permission list so we can restore world teams. + all_teams = await prisma.db.litellm_teamtable.find_many() + snapshot = {t.team_id: list(t.team_member_permissions or []) for t in all_teams} + + try: + seeder = world.keys[Actor.PROXY_ADMIN].cleartext + resp = await proxy_client.post( + "/team/permissions_bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"apply_to_all_teams": True, "permissions": [perm]}, + ) + assert resp.status_code == 200, resp.text + # `teams_updated` counts teams that didn't already have the perm — + # i.e. every team in the DB at call time. + assert resp.json()["teams_updated"] == len(all_teams) + + post = await prisma.db.litellm_teamtable.find_many() + for team in post: + perms = list(team.team_member_permissions or []) + assert perm in perms, f"team {team.team_id} missing the all-perm: {perms}" + # Existing perms preserved (merge, not replace). + for prior in snapshot.get(team.team_id, []): + assert ( + prior in perms + ), f"team {team.team_id} lost prior perm {prior!r}: {perms}" + finally: + # Restore every team — scratch teardown handles {a, b}; we must + # explicitly restore world teams (and any other non-scratch teams + # that snuck in) so downstream tests see the immutable world. + for team_id, prior_perms in snapshot.items(): + current = await prisma.db.litellm_teamtable.find_unique( + where={"team_id": team_id} + ) + if current is None: + continue + if list(current.team_member_permissions or []) != prior_perms: + await prisma.db.litellm_teamtable.update( + where={"team_id": team_id}, + data={"team_member_permissions": prior_perms}, + ) diff --git a/tests/proxy_behavior/management/test_team_update.py b/tests/proxy_behavior/management/test_team_update.py index 9b21911cef26..9cb2b0fecdaa 100644 --- a/tests/proxy_behavior/management/test_team_update.py +++ b/tests/proxy_behavior/management/test_team_update.py @@ -178,6 +178,24 @@ async def test_team_update_org_relocation_gate( assert row.organization_id == world.org_a_id, "denied but team relocated" +# Phase 4 F6 — explicit pin on the `_verify_team_access` 403 detail string +# when an org_admin clears the destination route gate but fails the source +# team's org-membership check. The relocation matrix above covers the +# status; this guard turns a silent rename of the helper's exception detail +# into a CI red. +async def test_team_update_org_b_admin_relocation_rejection_detail( + proxy_client, prisma, scratch, world +): + await _seed_target(prisma, world, "alpha", scratch.prefix) + resp = await proxy_client.post( + "/team/update", + headers={"Authorization": f"Bearer {world.keys[Actor.ORG_B_ADMIN].cleartext}"}, + json={"team_id": scratch.prefix, "organization_id": world.org_b_id}, + ) + assert resp.status_code == 403, resp.text + assert "do not have access to this team" in resp.text, resp.text + + async def test_team_update_org_relocation_allowed_for_dual_org_admin( proxy_client, prisma, scratch, world ): From 14c0a2b3e2c7320c7411c4e82f753e9b39a06ac2 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Sat, 23 May 2026 12:17:42 -0700 Subject: [PATCH 006/109] feat(prometheus): emit per-token-type detail metrics (LIT-3220) (#28372) (#28378) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(prometheus): emit per-token-type detail metrics (LIT-3220) (#28372) Adds five sparse counter metrics that break out the token detail fields providers already report in `usage.prompt_tokens_details` and `usage.completion_tokens_details`: - litellm_input_cached_tokens_metric (provider prompt-cache reads) - litellm_input_cache_creation_tokens_metric (Anthropic prompt-cache writes) - litellm_input_audio_tokens_metric (audio input tokens) - litellm_output_reasoning_tokens_metric (reasoning tokens) - litellm_output_audio_tokens_metric (audio output tokens) These are additive — existing input/output/total counters are unchanged, so no dashboards break. Each new counter is only incremented when the underlying detail is populated and > 0, keeping scrape output sparse for providers that don't report a given field. Data is read from the canonical Usage dict that `get_standard_logging_object_payload` already attaches at `standard_logging_payload["metadata"]["usage_object"]`, so no new plumbing through the logging pipeline is required. Tests: 10 new unit tests covering registration, label-set parity, all-types increment, zero/None/negative skip behaviour, and the no-metadata/no-usage_object no-op paths. Closes LIT-3220 Co-authored-by: shin-berri Co-authored-by: yuneng-jiang Co-authored-by: Krrish Dholakia Co-authored-by: Claude * chore: remove proof folder image --------- Co-authored-by: oss-agent-shin Co-authored-by: shin-berri Co-authored-by: yuneng-jiang Co-authored-by: Krrish Dholakia Co-authored-by: Claude Co-authored-by: Ishaan Jaffer --- litellm/integrations/prometheus.py | 142 ++++++++++ litellm/types/integrations/prometheus.py | 16 ++ .../test_prometheus_token_detail_metrics.py | 260 ++++++++++++++++++ 3 files changed, 418 insertions(+) create mode 100644 tests/test_litellm/integrations/test_prometheus_token_detail_metrics.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 2c63455565ce..5f0528421224 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -166,6 +166,53 @@ def __init__( # noqa: PLR0915 labelnames=self.get_labels_for_metric("litellm_output_tokens_metric"), ) + # Token-type detail metrics. These break out cached, cache-creation, + # audio and reasoning tokens that providers report inside + # prompt_tokens_details / completion_tokens_details on the usage + # object. They are sparse (only incremented when the provider + # reports a non-zero value) and are additive to the existing + # input/output token totals — no breaking change for existing + # dashboards built on the totals. + self.litellm_input_cached_tokens_metric = self._counter_factory( + "litellm_input_cached_tokens_metric", + "Provider-side cached input tokens (e.g. OpenAI prompt_tokens_details.cached_tokens, Anthropic cache_read_input_tokens)", + labelnames=self.get_labels_for_metric( + "litellm_input_cached_tokens_metric" + ), + ) + + self.litellm_input_cache_creation_tokens_metric = self._counter_factory( + "litellm_input_cache_creation_tokens_metric", + "Provider-side input tokens written to prompt cache (e.g. Anthropic cache_creation_input_tokens)", + labelnames=self.get_labels_for_metric( + "litellm_input_cache_creation_tokens_metric" + ), + ) + + self.litellm_input_audio_tokens_metric = self._counter_factory( + "litellm_input_audio_tokens_metric", + "Audio input tokens reported in prompt_tokens_details.audio_tokens", + labelnames=self.get_labels_for_metric( + "litellm_input_audio_tokens_metric" + ), + ) + + self.litellm_output_reasoning_tokens_metric = self._counter_factory( + "litellm_output_reasoning_tokens_metric", + "Reasoning tokens reported in completion_tokens_details.reasoning_tokens", + labelnames=self.get_labels_for_metric( + "litellm_output_reasoning_tokens_metric" + ), + ) + + self.litellm_output_audio_tokens_metric = self._counter_factory( + "litellm_output_audio_tokens_metric", + "Audio output tokens reported in completion_tokens_details.audio_tokens", + labelnames=self.get_labels_for_metric( + "litellm_output_audio_tokens_metric" + ), + ) + # Remaining Budget for Team self.litellm_remaining_team_budget_metric = self._gauge_factory( "litellm_remaining_team_budget_metric", @@ -1301,6 +1348,101 @@ def _increment_token_metrics( amount=float(standard_logging_payload["completion_tokens"]), ) + # Token-type detail metrics — sparse, only emitted when the provider + # reports a non-zero value in usage.prompt_tokens_details / + # usage.completion_tokens_details. + self._increment_token_detail_metrics( + standard_logging_payload=standard_logging_payload, + enum_values=enum_values, + label_context=label_context, + ) + + def _increment_token_detail_metrics( + self, + standard_logging_payload: StandardLoggingPayload, + enum_values: UserAPIKeyLabelValues, + label_context: Optional[PrometheusLabelFactoryContext] = None, + ) -> None: + """ + Increment per-token-type counters from the Usage object that providers + attach to the request. The Usage dict is plumbed onto + ``standard_logging_payload["metadata"]["usage_object"]`` by + ``get_standard_logging_object_payload``. + + Each counter is only incremented when the underlying value is > 0, so + scrape output stays sparse for providers that don't report these + details (most non-OpenAI/Anthropic models). + """ + metadata = standard_logging_payload.get("metadata") or {} + usage_object = ( + metadata.get("usage_object") if isinstance(metadata, dict) else None + ) + if not isinstance(usage_object, dict): + return + + prompt_details = usage_object.get("prompt_tokens_details") or {} + completion_details = usage_object.get("completion_tokens_details") or {} + + detail_metrics: List[Tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]] = [ + ( + self.litellm_input_cached_tokens_metric, + "litellm_input_cached_tokens_metric", + ( + prompt_details.get("cached_tokens") + if isinstance(prompt_details, dict) + else None + ), + ), + ( + self.litellm_input_cache_creation_tokens_metric, + "litellm_input_cache_creation_tokens_metric", + ( + prompt_details.get("cache_creation_tokens") + if isinstance(prompt_details, dict) + else None + ), + ), + ( + self.litellm_input_audio_tokens_metric, + "litellm_input_audio_tokens_metric", + ( + prompt_details.get("audio_tokens") + if isinstance(prompt_details, dict) + else None + ), + ), + ( + self.litellm_output_reasoning_tokens_metric, + "litellm_output_reasoning_tokens_metric", + ( + completion_details.get("reasoning_tokens") + if isinstance(completion_details, dict) + else None + ), + ), + ( + self.litellm_output_audio_tokens_metric, + "litellm_output_audio_tokens_metric", + ( + completion_details.get("audio_tokens") + if isinstance(completion_details, dict) + else None + ), + ), + ] + + for counter, metric_name, value in detail_metrics: + if not isinstance(value, (int, float)) or value <= 0: + continue + PrometheusLogger._inc_labeled_counter( + self, + counter, + metric_name, + enum_values, + label_context=label_context, + amount=float(value), + ) + def _increment_cache_metrics( self, standard_logging_payload: StandardLoggingPayload, diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 7b5c5ab2969e..827d10985cf8 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -201,6 +201,11 @@ class UserAPIKeyLabelNames(Enum): "litellm_total_tokens_metric", "litellm_input_tokens_metric", "litellm_output_tokens_metric", + "litellm_input_cached_tokens_metric", + "litellm_input_cache_creation_tokens_metric", + "litellm_input_audio_tokens_metric", + "litellm_output_reasoning_tokens_metric", + "litellm_output_audio_tokens_metric", "litellm_deployment_successful_fallbacks", "litellm_deployment_failed_fallbacks", "litellm_remaining_team_budget_metric", @@ -451,6 +456,17 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.MODEL_ID.value, ] + # Token-type detail metrics — reuse the same label set as + # litellm_input_tokens_metric / litellm_output_tokens_metric so dashboards + # can join across them. Only emitted when the underlying usage detail is + # populated by the provider (e.g. Anthropic cache_read_input_tokens, + # OpenAI prompt_tokens_details.cached_tokens, reasoning_tokens, audio_tokens). + litellm_input_cached_tokens_metric = litellm_input_tokens_metric + litellm_input_cache_creation_tokens_metric = litellm_input_tokens_metric + litellm_input_audio_tokens_metric = litellm_input_tokens_metric + litellm_output_reasoning_tokens_metric = litellm_output_tokens_metric + litellm_output_audio_tokens_metric = litellm_output_tokens_metric + litellm_deployment_state = [ UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.MODEL_ID.value, diff --git a/tests/test_litellm/integrations/test_prometheus_token_detail_metrics.py b/tests/test_litellm/integrations/test_prometheus_token_detail_metrics.py new file mode 100644 index 000000000000..72a4e80717b1 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_token_detail_metrics.py @@ -0,0 +1,260 @@ +""" +Unit tests for the per-token-type Prometheus detail metrics added for LIT-3220. + +These metrics break out cached, cache-creation, audio and reasoning tokens +from the Usage object that providers report. They are sparse — only +incremented when the underlying detail is populated and > 0. + +Run with: + uv run pytest tests/test_litellm/integrations/test_prometheus_token_detail_metrics.py -v +""" + +from typing import get_args +from unittest.mock import MagicMock + +import pytest + +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.integrations.prometheus import ( + DEFINED_PROMETHEUS_METRICS, + PrometheusMetricLabels, + UserAPIKeyLabelValues, +) + + +TOKEN_DETAIL_METRICS = [ + "litellm_input_cached_tokens_metric", + "litellm_input_cache_creation_tokens_metric", + "litellm_input_audio_tokens_metric", + "litellm_output_reasoning_tokens_metric", + "litellm_output_audio_tokens_metric", +] + + +@pytest.fixture +def sample_enum_values(): + return UserAPIKeyLabelValues( + end_user="test-end-user", + hashed_api_key="test-key-hash", + api_key_alias="test-key-alias", + team="test-team", + team_alias="test-team-alias", + user="test-user", + model="gpt-4o", + ) + + +def _make_mock_logger(): + """Mock instance with the five detail counters + get_labels_for_metric.""" + logger = MagicMock() + for name in TOKEN_DETAIL_METRICS: + setattr(logger, name, MagicMock()) + logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + return logger + + +class TestTokenDetailMetricsRegistration: + """Metric registration / wiring — no runtime needed.""" + + def test_metrics_in_defined_prometheus_metrics(self): + defined = get_args(DEFINED_PROMETHEUS_METRICS) + for name in TOKEN_DETAIL_METRICS: + assert name in defined, f"{name} missing from DEFINED_PROMETHEUS_METRICS" + + def test_metric_labels_defined(self): + for name in TOKEN_DETAIL_METRICS: + assert hasattr( + PrometheusMetricLabels, name + ), f"{name} missing from PrometheusMetricLabels" + + def test_input_detail_metrics_share_input_label_set(self): + # Detail metrics should reuse the parent input/output label set so + # dashboards can join token totals against per-type detail. + assert ( + PrometheusMetricLabels.litellm_input_cached_tokens_metric + == PrometheusMetricLabels.litellm_input_tokens_metric + ) + assert ( + PrometheusMetricLabels.litellm_input_cache_creation_tokens_metric + == PrometheusMetricLabels.litellm_input_tokens_metric + ) + assert ( + PrometheusMetricLabels.litellm_input_audio_tokens_metric + == PrometheusMetricLabels.litellm_input_tokens_metric + ) + + def test_output_detail_metrics_share_output_label_set(self): + assert ( + PrometheusMetricLabels.litellm_output_reasoning_tokens_metric + == PrometheusMetricLabels.litellm_output_tokens_metric + ) + assert ( + PrometheusMetricLabels.litellm_output_audio_tokens_metric + == PrometheusMetricLabels.litellm_output_tokens_metric + ) + + +class TestIncrementTokenDetailMetrics: + """Behaviour of PrometheusLogger._increment_token_detail_metrics.""" + + def test_increments_all_present_token_types(self, sample_enum_values): + logger = _make_mock_logger() + payload = { + "metadata": { + "usage_object": { + "prompt_tokens": 100, + "completion_tokens": 80, + "total_tokens": 180, + "prompt_tokens_details": { + "cached_tokens": 40, + "cache_creation_tokens": 25, + "audio_tokens": 15, + }, + "completion_tokens_details": { + "reasoning_tokens": 60, + "audio_tokens": 10, + }, + } + }, + } + + PrometheusLogger._increment_token_detail_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + logger.litellm_input_cached_tokens_metric.labels().inc.assert_called_once_with( + 40.0 + ) + logger.litellm_input_cache_creation_tokens_metric.labels().inc.assert_called_once_with( + 25.0 + ) + logger.litellm_input_audio_tokens_metric.labels().inc.assert_called_once_with( + 15.0 + ) + logger.litellm_output_reasoning_tokens_metric.labels().inc.assert_called_once_with( + 60.0 + ) + logger.litellm_output_audio_tokens_metric.labels().inc.assert_called_once_with( + 10.0 + ) + + def test_skips_metrics_when_value_is_zero(self, sample_enum_values): + logger = _make_mock_logger() + payload = { + "metadata": { + "usage_object": { + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_creation_tokens": 0, + "audio_tokens": 0, + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + }, + } + } + } + + PrometheusLogger._increment_token_detail_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + for name in TOKEN_DETAIL_METRICS: + getattr(logger, name).labels.assert_not_called() + + def test_skips_metrics_when_value_is_none(self, sample_enum_values): + logger = _make_mock_logger() + payload = { + "metadata": { + "usage_object": { + "prompt_tokens_details": { + "cached_tokens": None, + "audio_tokens": 12, + }, + "completion_tokens_details": {}, + } + } + } + + PrometheusLogger._increment_token_detail_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + # Only audio_tokens was non-zero — only that counter should fire. + logger.litellm_input_cached_tokens_metric.labels.assert_not_called() + logger.litellm_input_cache_creation_tokens_metric.labels.assert_not_called() + logger.litellm_input_audio_tokens_metric.labels().inc.assert_called_once_with( + 12.0 + ) + logger.litellm_output_reasoning_tokens_metric.labels.assert_not_called() + logger.litellm_output_audio_tokens_metric.labels.assert_not_called() + + def test_no_usage_object_is_a_noop(self, sample_enum_values): + logger = _make_mock_logger() + payload = {"metadata": {}} + + # Should not raise and should not call any counter. + PrometheusLogger._increment_token_detail_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + for name in TOKEN_DETAIL_METRICS: + getattr(logger, name).labels.assert_not_called() + + def test_missing_metadata_is_a_noop(self, sample_enum_values): + logger = _make_mock_logger() + + # Many error / cache-hit paths leave metadata as None. + PrometheusLogger._increment_token_detail_metrics( + logger, + standard_logging_payload={"metadata": None}, # type: ignore[typeddict-item] + enum_values=sample_enum_values, + ) + + for name in TOKEN_DETAIL_METRICS: + getattr(logger, name).labels.assert_not_called() + + def test_negative_values_are_ignored(self, sample_enum_values): + # Defensive: a buggy upstream that returned a negative shouldn't + # poison the counter (counters can't go down without a reset). + logger = _make_mock_logger() + payload = { + "metadata": { + "usage_object": { + "prompt_tokens_details": {"cached_tokens": -5}, + "completion_tokens_details": {"reasoning_tokens": -10}, + } + } + } + + PrometheusLogger._increment_token_detail_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + logger.litellm_input_cached_tokens_metric.labels.assert_not_called() + logger.litellm_output_reasoning_tokens_metric.labels.assert_not_called() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 886e91b85e8c651d95799b7be8b25ebac16cac35 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 23 May 2026 12:21:32 -0700 Subject: [PATCH 007/109] fix(otel): stamp http.response.status_code on all error responses (#28405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(otel): stamp http.response.status_code on all error responses httpx.HTTPStatusError exposes status under .response.status_code, not as a top-level attr, so unified-endpoint 5xx failures left the SERVER span without a status. The admin hooks only wrote a child span and never stamped or ended the parent at all, so admin 4xx/5xx (and success) responses were invisible to dashboards. Adds a fallback to .response.status_code in get_error_information, and ends the parent SERVER span in async_management_endpoint_{success,failure}_hook with the same _record_exception_on_span helper the unified path uses. Resolves LIT-3193 * test(otel): exercise httpx.HTTPStatusError through admin path Pins the contract that get_error_information's response.status_code fallback is reachable from any entry point — without this, a future refactor that bypasses _record_exception_on_span in the admin hooks could regress for httpx-wrapped exceptions while the unified suite still passes. * chore(otel): trim verbose comments in LIT-3193 changes Tighten docstrings and remove redundant section dividers/inline narration. Behavior is unchanged. * fix(otel): set span.status on management hook parent SERVER span Mirror the unified failure path: stamp StatusCode.ERROR on the parent SERVER span before recording the exception, and StatusCode.OK before ending it on success. Without this, OTEL backends filtering on span status (the idiomatic primitive) miss admin-endpoint failures even though the http.response.status_code attribute is correct. Extend assert_server_span_attrs to assert span.status.status_code matches the expected outcome so the gap can't regress. * fix(otel): close SERVER span on body-validation and unhandled errors Stash the SERVER span on request.state in auth so FastAPI exception handlers can finish it for failures that occur after auth but before the route handler (e.g. /model/new TypeError, /key/generate RequestValidationError). Without this, those requests left dangling spans missing http.response.status_code. Resolves LIT-3193 * fix(otel): generic 500 body, log exception details server-side Don't leak str(exc) and type(exc).__name__ to clients on uncaught exceptions. The full traceback is logged via verbose_proxy_logger and the SERVER span still gets http.response.status_code=500. Resolves LIT-3193 * fix(otel): stamp http.response.status_code on every SERVER span path Closes three remaining gaps where the proxy SERVER span ended without the http.response.status_code attribute: 1. ProxyException raised from _read_request_body (e.g. invalid JSON body) bubbled out of user_api_key_auth before the SERVER span was created, so the FastAPI handler had nothing to close and the trace never reached the backend. Hoist the span creation to a new idempotent _ensure_parent_otel_span_on_request_state helper called at the top of user_api_key_auth; wire openai_exception_handler to close the dangling span. Covers /v1/chat/completions, /v1/messages, /v1/responses (shared handler). 2. /v1/responses success — _handle_success ends the proxy span before async_post_call_success_hook fires on this path, so the hook's set_response_status_code_attribute(200) silently no-op'd against an ended span. Stamp 200 + set OK status at the close site in _handle_success / _end_proxy_span_from_kwargs via a shared _close_proxy_span_ok helper, so the attribute lands regardless of which success hook runs first. 3. Failure path for exceptions without code/status_code (e.g. a bare TypeError surfacing through _handle_llm_api_exception) — empty error_information.error_code → _record_exception_on_span skips the stamp → the hook ends the span. Default to 500 in async_post_call_failure_hook so the attribute is always set. Resolves LIT-3193 --- litellm/integrations/opentelemetry.py | 50 +++- litellm/litellm_core_utils/litellm_logging.py | 8 +- litellm/proxy/auth/user_api_key_auth.py | 61 +++-- .../key_management_endpoints.py | 1 + litellm/proxy/management_helpers/utils.py | 30 ++- litellm/proxy/proxy_server.py | 61 ++++- .../integrations/open_telemetry/__init__.py | 0 .../integrations/open_telemetry/_helpers.py | 109 +++++++++ .../integrations/open_telemetry/conftest.py | 93 ++++++++ .../test_otel_admin_endpoints.py | 182 ++++++++++++++ .../test_otel_exception_handler.py | 136 +++++++++++ .../test_otel_passthrough_endpoints.py | 136 +++++++++++ .../test_otel_unified_endpoints.py | 223 ++++++++++++++++++ 13 files changed, 1050 insertions(+), 40 deletions(-) create mode 100644 tests/test_litellm/integrations/open_telemetry/__init__.py create mode 100644 tests/test_litellm/integrations/open_telemetry/_helpers.py create mode 100644 tests/test_litellm/integrations/open_telemetry/conftest.py create mode 100644 tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py create mode 100644 tests/test_litellm/integrations/open_telemetry/test_otel_exception_handler.py create mode 100644 tests/test_litellm/integrations/open_telemetry/test_otel_passthrough_endpoints.py create mode 100644 tests/test_litellm/integrations/open_telemetry/test_otel_unified_endpoints.py diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 6c8510380a84..81fdc5a1e219 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -702,6 +702,14 @@ async def async_post_call_failure_hook( }, ) + # _record_exception_on_span only stamps when error_code is set; + # bare TypeError etc. has none, and the span is about to be ended. + error_code = ( + error_information.get("error_code") if error_information else None + ) + if not error_code: + self.set_response_status_code_attribute(parent_otel_span, 500) + # Pre-request latency (request_data carries the propagated # metadata on the failure path; omitted if it failed before handoff). self.set_preprocessing_duration_attribute(parent_otel_span, request_data) @@ -798,11 +806,6 @@ async def async_post_call_success_hook( # Pre-request latency on the SERVER span (success path). self.set_preprocessing_duration_attribute(parent_span, kwargs) - # http.response.status_code on the SERVER span (success path). - # A successful proxy response is HTTP 200; the failure path sets - # this from the error code in _record_exception_on_span. - self.set_response_status_code_attribute(parent_span, 200) - # 3. Guardrail span self._create_guardrail_span(kwargs=kwargs, context=ctx) @@ -985,7 +988,15 @@ def _end_proxy_span_from_kwargs(self, kwargs: dict, end_time) -> None: and hasattr(proxy_span, "is_recording") and proxy_span.is_recording() ): - proxy_span.end(end_time=self._to_ns(end_time)) + self._close_proxy_span_ok(proxy_span, end_time) + + def _close_proxy_span_ok(self, span: Span, end_time) -> None: + """Stamp http.response.status_code=200 + status=OK, then end the span.""" + from opentelemetry.trace import Status, StatusCode + + self.set_response_status_code_attribute(span, 200) + span.set_status(Status(StatusCode.OK)) + span.end(end_time=self._to_ns(end_time)) def _handle_success(self, kwargs, response_obj, start_time, end_time): """Create the litellm_request span then close the proxy span.""" @@ -1071,8 +1082,10 @@ def _handle_success(self, kwargs, response_obj, start_time, end_time): parent_span is not None and hasattr(parent_span, "name") and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME + and hasattr(parent_span, "is_recording") + and parent_span.is_recording() ): - parent_span.end(end_time=self._to_ns(end_time)) + self._close_proxy_span_ok(parent_span, end_time) # Stamp team attributes onto the SERVER (root) span before it is # closed, so the trace root carries them like every child span. @@ -3041,6 +3054,11 @@ async def async_management_endpoint_success_hook( management_endpoint_span.set_status(Status(StatusCode.OK)) management_endpoint_span.end(end_time=_end_time_ns) + # The management wrapper has no other hook that closes the SERVER span. + self.set_response_status_code_attribute(parent_otel_span, 200) + parent_otel_span.set_status(Status(StatusCode.OK)) + parent_otel_span.end(end_time=_end_time_ns) + async def async_management_endpoint_failure_hook( self, logging_payload: ManagementEndpointLoggingPayload, @@ -3091,6 +3109,24 @@ async def async_management_endpoint_failure_hook( management_endpoint_span.set_status(Status(StatusCode.ERROR)) management_endpoint_span.end(end_time=_end_time_ns) + # The management wrapper has no other hook that closes the SERVER span. + from litellm.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup, + ) + + error_information = StandardLoggingPayloadSetup.get_error_information( + original_exception=_exception, + ) + parent_otel_span.set_status(Status(StatusCode.ERROR)) + self._record_exception_on_span( + span=parent_otel_span, + kwargs={ + "exception": _exception, + "standard_logging_object": {"error_information": error_information}, + }, + ) + parent_otel_span.end(end_time=_end_time_ns) + def create_litellm_proxy_request_started_span( self, start_time: datetime, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2ab037afb0d5..63fa0e64695f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5140,13 +5140,17 @@ def get_error_information( ) -> StandardLoggingPayloadErrorInformation: from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG - # Check for 'code' first (used by ProxyException), then fall back to 'status_code' (used by LiteLLM exceptions) - # Ensure error_code is always a string for Prisma Python JSON field compatibility + # ProxyException uses .code, LiteLLM exceptions use .status_code, + # httpx.HTTPStatusError exposes status only as .response.status_code. + # Stringified for Prisma JSON compatibility. error_code_attr = getattr(original_exception, "code", None) if error_code_attr is not None and str(error_code_attr) not in ("", "None"): error_status: str = str(error_code_attr) else: status_code_attr = getattr(original_exception, "status_code", None) + if status_code_attr is None: + response_attr = getattr(original_exception, "response", None) + status_code_attr = getattr(response_attr, "status_code", None) error_status = str(status_code_attr) if status_code_attr is not None else "" error_class: str = ( str(original_exception.__class__.__name__) if original_exception else "" diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 6974860a22a4..032786339280 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -671,6 +671,37 @@ async def _resolve_jwt_to_virtual_key( return None +def _ensure_parent_otel_span_on_request_state(request: Request) -> None: + """Idempotently create the OTEL SERVER span and stash it on + ``request.state.parent_otel_span``. Safe to call multiple times. + + Called both at the top of ``user_api_key_auth`` (so body-parse failures + have a span to close) and inside ``_user_api_key_auth_builder`` (for + callers that bypass ``user_api_key_auth``, e.g. MCP). + """ + from litellm.proxy.proxy_server import open_telemetry_logger + + if open_telemetry_logger is None: + return + if getattr(request.state, "parent_otel_span", None) is not None: + return + start_time = datetime.now() + try: + request.state.litellm_received_at = start_time + except Exception: + pass + parent_otel_span = open_telemetry_logger.create_litellm_proxy_request_started_span( + start_time=start_time, + headers=_safe_get_request_headers(request), + ) + open_telemetry_logger.set_proxy_request_route_attributes( + parent_otel_span, + url_path=get_request_route(request=request), + http_route=get_request_route_template(request), + ) + request.state.parent_otel_span = parent_otel_span + + async def _user_api_key_auth_builder( # noqa: PLR0915 request: Request, api_key: str, @@ -697,9 +728,10 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) parent_otel_span: Optional[Span] = None - start_time = datetime.now() - # Stash the proxy-receive instant for the pre-request latency calc — - # the OTel Span API exposes no start-time getter, so propagate it. + # Prefer the receive-instant stamped by the early helper in + # user_api_key_auth (before body parse) — overwriting it would shorten + # the preprocessing-duration measurement by the body-parse window. + start_time = getattr(request.state, "litellm_received_at", None) or datetime.now() try: request.state.litellm_received_at = start_time except Exception: @@ -739,18 +771,11 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) if open_telemetry_logger is not None: - parent_otel_span = ( - open_telemetry_logger.create_litellm_proxy_request_started_span( - start_time=start_time, - headers=_safe_get_request_headers(request), - ) - ) - # `route` is the literal path; template from the matched route. - open_telemetry_logger.set_proxy_request_route_attributes( - parent_otel_span, - url_path=route, - http_route=get_request_route_template(request), - ) + # Reuse the span created by user_api_key_auth (before body parse) + # so it survives _read_request_body failures. For callers that + # bypass user_api_key_auth (e.g. MCP), create it lazily. + _ensure_parent_otel_span_on_request_state(request) + parent_otel_span = getattr(request.state, "parent_otel_span", None) ### USER-DEFINED AUTH FUNCTION ### if enterprise_custom_auth is not None: @@ -2149,6 +2174,12 @@ async def user_api_key_auth( Parent function to authenticate user api key / jwt token. """ + # Create the SERVER span and stash it on request.state BEFORE reading the + # body. _read_request_body can raise ProxyException for malformed JSON; + # without this, that path leaves no span for the exception handler to + # close, and the trace never reaches the backend. + _ensure_parent_otel_span_on_request_state(request) + request_data = await _read_request_body(request=request) request_data = populate_request_with_path_params( request_data=request_data, request=request diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index eebcf0b65714..a67d8d934bff 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3226,6 +3226,7 @@ async def info_key_fn_v2( @router.get( "/key/info", tags=["key management"], dependencies=[Depends(user_api_key_auth)] ) +@management_endpoint_wrapper async def info_key_fn( key: Optional[str] = fastapi.Query( default=None, description="Key in the request parameters" diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index f2d6e9612ffe..b7d5cc30c492 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -518,19 +518,23 @@ async def wrapper(*args, **kwargs): _request_body: dict = await _read_request_body( request=_http_request ) - logging_payload = ManagementEndpointLoggingPayload( - route=_route, - request_data=_request_body, - response=None, - start_time=start_time, - end_time=end_time, - exception=e, - ) - - await open_telemetry_logger.async_management_endpoint_failure_hook( # type: ignore - logging_payload=logging_payload, - parent_otel_span=parent_otel_span, - ) + else: + _route = func.__name__ + _request_body = {} + + logging_payload = ManagementEndpointLoggingPayload( + route=_route, + request_data=_request_body, + response=None, + start_time=start_time, + end_time=end_time, + exception=e, + ) + + await open_telemetry_logger.async_management_endpoint_failure_hook( # type: ignore + logging_payload=logging_payload, + parent_otel_span=parent_otel_span, + ) raise e diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 759534a32a1f..4af7caead0ef 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -550,6 +550,7 @@ def generate_feedback_box(): status, ) from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.openapi.docs import get_swagger_ui_html from fastapi.openapi.utils import get_openapi @@ -1209,15 +1210,69 @@ async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions headers = exc.headers error_dict = exc.to_dict() + status_code = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR + _close_dangling_otel_server_span(request, status_code) return JSONResponse( - status_code=( - int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR - ), + status_code=status_code, content={"error": error_dict}, headers=headers, ) +def _close_dangling_otel_server_span(request: Request, status_code: int) -> None: + parent_otel_span = getattr(request.state, "parent_otel_span", None) + if parent_otel_span is None: + return + if open_telemetry_logger is None: + return + try: + from opentelemetry.trace import Status, StatusCode + + open_telemetry_logger.set_response_status_code_attribute( + parent_otel_span, status_code + ) + parent_otel_span.set_status( + Status(StatusCode.ERROR if status_code >= 400 else StatusCode.OK) + ) + parent_otel_span.end() + except Exception as e: + verbose_proxy_logger.debug( + "Error closing dangling OTEL SERVER span: %s", str(e) + ) + finally: + request.state.parent_otel_span = None + + +@app.exception_handler(RequestValidationError) +async def otel_request_validation_exception_handler( + request: Request, exc: RequestValidationError +): + _close_dangling_otel_server_span(request, 422) + return JSONResponse( + status_code=422, + content={"detail": jsonable_encoder(exc.errors())}, + ) + + +@app.exception_handler(Exception) +async def otel_unhandled_exception_handler(request: Request, exc: Exception): + if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)): + raise exc + verbose_proxy_logger.exception( + "Unhandled exception in request: %s", type(exc).__name__ + ) + _close_dangling_otel_server_span(request, 500) + return JSONResponse( + status_code=500, + content={ + "error": { + "message": "Internal server error", + "type": "internal_server_error", + } + }, + ) + + router = APIRouter() diff --git a/tests/test_litellm/integrations/open_telemetry/__init__.py b/tests/test_litellm/integrations/open_telemetry/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/test_litellm/integrations/open_telemetry/_helpers.py b/tests/test_litellm/integrations/open_telemetry/_helpers.py new file mode 100644 index 000000000000..69e9daf67cbb --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/_helpers.py @@ -0,0 +1,109 @@ +""" +Helpers for the LIT-3193 OTEL HTTP-attribute matrix. + +Module split from ``conftest.py`` because pytest auto-discovers fixtures but +forbids ``from .conftest import …`` (no parent package). Fixtures stay in +``conftest.py``; pure helpers (assertions, exception factories, attribute +constants) live here so test modules can ``from ._helpers import …``. +""" + +from typing import Any, Optional + +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import StatusCode + +from litellm.integrations.opentelemetry import ( + HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, + HTTP_ROUTE_ATTRIBUTE, + LITELLM_PROXY_REQUEST_SPAN_NAME, + URL_PATH_ATTRIBUTE, +) + + +def get_server_span(exporter: InMemorySpanExporter): + """Return the (single) finished SERVER span, or None if it never ended.""" + for s in exporter.get_finished_spans(): + if s.name == LITELLM_PROXY_REQUEST_SPAN_NAME: + return s + return None + + +def assert_server_span_attrs( + exporter: InMemorySpanExporter, + *, + expected_status: int, + expected_url_path: str, + expected_http_route: Optional[str] = None, + where: str = "", +) -> None: + """The four required attributes on the SERVER span must all be set.""" + span = get_server_span(exporter) + assert span is not None, ( + f"{where}: SERVER span never finished — exporter saw " + f"{[s.name for s in exporter.get_finished_spans()]}" + ) + + actual_status = span.attributes.get(HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE) + assert actual_status == expected_status, ( + f"{where}: {HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE} = " + f"{actual_status!r}, expected {expected_status}" + ) + assert isinstance( + actual_status, int + ), f"{where}: status code must be int (semconv), got {type(actual_status)}" + + actual_url = span.attributes.get(URL_PATH_ATTRIBUTE) + assert actual_url == expected_url_path, ( + f"{where}: {URL_PATH_ATTRIBUTE} = {actual_url!r}, " + f"expected {expected_url_path!r}" + ) + + expected_route = expected_http_route or expected_url_path + actual_route = span.attributes.get(HTTP_ROUTE_ATTRIBUTE) + assert actual_route == expected_route, ( + f"{where}: {HTTP_ROUTE_ATTRIBUTE} = {actual_route!r}, " + f"expected {expected_route!r}" + ) + + duration_ns = (span.end_time or 0) - (span.start_time or 0) + assert duration_ns > 0, f"{where}: duration must be > 0, got {duration_ns}ns" + + expected_span_status = StatusCode.ERROR if expected_status >= 400 else StatusCode.OK + actual_span_status = span.status.status_code + assert actual_span_status == expected_span_status, ( + f"{where}: span.status = {actual_span_status!r}, " + f"expected {expected_span_status!r}" + ) + + +# --------------------------------------------------------------------------- +# Synthetic exceptions covering the matrix triggers +# --------------------------------------------------------------------------- +class HttpStatusException(Exception): + """Generic exception with .status_code; mirrors what proxy code reads.""" + + def __init__(self, status_code: int, message: str = "boom"): + super().__init__(message) + self.status_code = status_code + self.code = status_code + + +def make_httpx_status_error(status_code: int, body: str = "upstream error"): + """Real httpx.HTTPStatusError — what providers emit on 4xx/5xx upstream.""" + import httpx + + request = httpx.Request("POST", "https://upstream.example/v1/x") + response = httpx.Response( + status_code=status_code, content=body.encode("utf-8"), request=request + ) + return httpx.HTTPStatusError( + f"HTTP {status_code}", request=request, response=response + ) + + +def make_fastapi_http_exception(status_code: int, detail: Any = "boom"): + from fastapi import HTTPException + + return HTTPException(status_code=status_code, detail=detail) diff --git a/tests/test_litellm/integrations/open_telemetry/conftest.py b/tests/test_litellm/integrations/open_telemetry/conftest.py new file mode 100644 index 000000000000..b29335aedd8f --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/conftest.py @@ -0,0 +1,93 @@ +""" +Shared fixtures for the LIT-3193 OTEL HTTP-attribute matrix. + +The matrix needs every error response — across unified inference, passthrough, +and admin endpoints — to carry ``http.response.status_code``, ``url.path``, +``http.route``, and a non-zero duration on the SERVER (root) span. These +fixtures hook a real ``OpenTelemetry`` callback into ``litellm.callbacks`` so +the tests drive the actual handler / wrapper code under test, not the OTEL +emitter in isolation. + +See ``LIT-3193_test_matrix.md`` (same directory) for the cell list. +""" + +import os +import sys +from datetime import datetime +from typing import Optional, Tuple +from unittest.mock import MagicMock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.integrations.opentelemetry import OpenTelemetry + + +# --------------------------------------------------------------------------- +# OTEL + exporter +# --------------------------------------------------------------------------- +@pytest.fixture +def otel_with_exporter() -> Tuple[OpenTelemetry, InMemorySpanExporter]: + """Real OpenTelemetry callback with every span captured in-memory.""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + + otel = OpenTelemetry() + otel.tracer = provider.get_tracer("lit-3193-tests") + otel.message_logging = True + return otel, exporter + + +@pytest.fixture +def server_span_factory(otel_with_exporter): + """Factory mirroring user_api_key_auth: SERVER span + url.path + http.route.""" + otel, _exporter = otel_with_exporter + + def _make(url_path: str, http_route: Optional[str] = None): + span = otel.create_litellm_proxy_request_started_span( + start_time=datetime.now(), headers={} + ) + otel.set_proxy_request_route_attributes( + span, url_path=url_path, http_route=http_route or url_path + ) + return span + + return _make + + +@pytest.fixture +def user_api_key_dict_factory(): + """UserAPIKeyAuth-shaped mock; the only attr the failure hooks read is + parent_otel_span (plus team_id/team_alias for stamping).""" + + def _make(parent_span): + d = MagicMock() + d.parent_otel_span = parent_span + d.team_id = "team-lit-3193" + d.team_alias = "lit-3193-team" + d.request_route = None + return d + + return _make + + +@pytest.fixture +def register_otel_callback(otel_with_exporter, monkeypatch): + """Make ProxyLogging.post_call_failure_hook iterate our OTEL instance.""" + otel, _ = otel_with_exporter + saved = list(litellm.callbacks) + monkeypatch.setattr(litellm, "callbacks", [otel]) + yield otel + litellm.callbacks = saved + + +# Helpers (assertions, exception factories) live in ``_helpers.py`` — pytest +# auto-discovers fixtures here but forbids ``from .conftest import …``. diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py new file mode 100644 index 000000000000..b1a3b834c3da --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -0,0 +1,182 @@ +"""LIT-3193 — admin / management endpoints. Drives the +async_management_endpoint_{success,failure}_hook integration points.""" + +import asyncio +from datetime import datetime + +import pytest + +from litellm.proxy._types import ( + ManagementEndpointLoggingPayload, + UserAPIKeyAuth, +) + +from ._helpers import ( + HttpStatusException, + assert_server_span_attrs, + make_fastapi_http_exception, + make_httpx_status_error, +) + + +def _real_user_api_key_dict(parent_span): + return UserAPIKeyAuth( + api_key="sk-test-admin", + team_id="team-lit-3193", + team_alias="lit-3193-team", + parent_otel_span=parent_span, + ) + + +async def _drive_admin_failure(*, otel, exception, parent_span, route): + payload = ManagementEndpointLoggingPayload( + route=route, + request_data={}, + response=None, + start_time=datetime.now(), + end_time=datetime.now(), + exception=exception, + ) + await otel.async_management_endpoint_failure_hook( + logging_payload=payload, + parent_otel_span=parent_span, + ) + + +async def _drive_admin_success(*, otel, parent_span, route, response): + payload = ManagementEndpointLoggingPayload( + route=route, + request_data={}, + response=response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + await otel.async_management_endpoint_success_hook( + logging_payload=payload, + parent_otel_span=parent_span, + ) + + +KEY_GENERATE_PATH = "/key/generate" + + +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(400, "negative max_budget"), 400), + (make_fastapi_http_exception(401, "missing master key"), 401), + (make_fastapi_http_exception(403, "non-admin"), 403), + (make_fastapi_http_exception(422, "validation"), 422), + (HttpStatusException(500, "DB unreachable"), 500), + # Pins .response.status_code fallback through the admin path. + (make_httpx_status_error(500, "upstream blew up"), 500), + ], + ids=["400", "401", "403", "422", "500", "500-httpx"], +) +def test_key_generate_failure_stamps_server_span( + exception, + expected_status, + server_span_factory, + otel_with_exporter, +): + otel, exporter = otel_with_exporter + server_span = server_span_factory(KEY_GENERATE_PATH) + + asyncio.run( + _drive_admin_failure( + otel=otel, + exception=exception, + parent_span=server_span, + route=KEY_GENERATE_PATH, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=KEY_GENERATE_PATH, + where=f"key/generate {expected_status}", + ) + + +def test_key_generate_success_stamps_server_span( + server_span_factory, otel_with_exporter +): + otel, exporter = otel_with_exporter + server_span = server_span_factory(KEY_GENERATE_PATH) + + asyncio.run( + _drive_admin_success( + otel=otel, + parent_span=server_span, + route=KEY_GENERATE_PATH, + response={"key": "sk-1", "key_name": "k"}, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=KEY_GENERATE_PATH, + where="key/generate 200", + ) + + +SMOKE_ADMIN_ENDPOINTS = [ + "/key/info", + "/key/update", + "/key/delete", + "/team/new", + "/team/member_add", + "/user/new", + "/user/info", + "/model/new", + "/model/delete", + "/customer/new", + "/customer/info", + "/organization/new", + "/organization/member_add", + "/budget/new", + "/budget/info", + "/credentials/new", + "/mcp/server/add", + "/tag/new", +] + + +@pytest.mark.parametrize("path", SMOKE_ADMIN_ENDPOINTS) +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(404, "not found"), 404), + (HttpStatusException(500, "DB unreachable"), 500), + ], + ids=["404", "500"], +) +def test_admin_endpoint_failure_stamps_server_span( + path, + exception, + expected_status, + server_span_factory, + otel_with_exporter, +): + """Confirm SERVER-span stamping works for every admin resource family — + same wrapper, just different routes.""" + otel, exporter = otel_with_exporter + server_span = server_span_factory(path) + + asyncio.run( + _drive_admin_failure( + otel=otel, + exception=exception, + parent_span=server_span, + route=path, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=path, + where=f"{path} {expected_status}", + ) diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_exception_handler.py b/tests/test_litellm/integrations/open_telemetry/test_otel_exception_handler.py new file mode 100644 index 000000000000..56059c260c3d --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_exception_handler.py @@ -0,0 +1,136 @@ +"""LIT-3193 — exception-handler path. Closes SERVER spans for requests +that fail after auth but before the route handler runs (e.g. /model/new +TypeError or RequestValidationError).""" + +import asyncio +import types + +import pytest +from fastapi import HTTPException +from fastapi.exceptions import RequestValidationError + +import litellm.proxy.proxy_server as proxy_server_module +from litellm.proxy._types import ProxyException +from litellm.proxy.proxy_server import ( + _close_dangling_otel_server_span, + openai_exception_handler, + otel_request_validation_exception_handler, + otel_unhandled_exception_handler, +) + +from ._helpers import assert_server_span_attrs + + +def _fake_request(parent_otel_span=None): + state = types.SimpleNamespace() + if parent_otel_span is not None: + state.parent_otel_span = parent_otel_span + return types.SimpleNamespace(state=state) + + +@pytest.fixture +def wired_otel(otel_with_exporter, monkeypatch): + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server_module, "open_telemetry_logger", otel) + return exporter + + +@pytest.mark.parametrize("status,path", [(500, "/model/new"), (422, "/key/generate")]) +def test_close_dangling_span_stamps_status( + wired_otel, server_span_factory, status, path +): + request = _fake_request(parent_otel_span=server_span_factory(path)) + _close_dangling_otel_server_span(request, status) + assert_server_span_attrs( + wired_otel, + expected_status=status, + expected_url_path=path, + where=f"{path} {status}", + ) + assert request.state.parent_otel_span is None + + +def test_close_dangling_span_noop_when_no_span(wired_otel): + _close_dangling_otel_server_span(_fake_request(), 500) + assert wired_otel.get_finished_spans() == () + + +def test_close_dangling_span_noop_when_otel_absent(server_span_factory, monkeypatch): + monkeypatch.setattr(proxy_server_module, "open_telemetry_logger", None) + request = _fake_request(parent_otel_span=server_span_factory("/key/generate")) + _close_dangling_otel_server_span(request, 500) + + +@pytest.mark.parametrize( + "handler,exc,status,path", + [ + ( + otel_request_validation_exception_handler, + RequestValidationError(errors=[]), + 422, + "/key/generate", + ), + ( + otel_unhandled_exception_handler, + TypeError("Deployment.__init__() missing required positional arg"), + 500, + "/model/new", + ), + ], +) +def test_exception_handler_closes_span( + wired_otel, server_span_factory, handler, exc, status, path +): + request = _fake_request(parent_otel_span=server_span_factory(path)) + response = asyncio.run(handler(request, exc)) + assert response.status_code == status + assert_server_span_attrs( + wired_otel, + expected_status=status, + expected_url_path=path, + where=f"{handler.__name__} ({type(exc).__name__})", + ) + + +def test_unhandled_handler_reraises_known_exceptions(wired_otel, server_span_factory): + """ProxyException / HTTPException / RequestValidationError have dedicated handlers.""" + request = _fake_request(parent_otel_span=server_span_factory("/key/generate")) + with pytest.raises(HTTPException): + asyncio.run( + otel_unhandled_exception_handler( + request, HTTPException(status_code=403, detail="forbidden") + ) + ) + + +# Covers ProxyException raised after auth stashed the span (e.g., invalid-JSON +# body via _read_request_body) — handler must close the dangling SERVER span. +@pytest.mark.parametrize( + "code,path", + [ + (400, "/v1/chat/completions"), + (400, "/v1/messages"), + (400, "/v1/responses"), + (429, "/v1/chat/completions"), + (503, "/v1/chat/completions"), + ], +) +def test_openai_exception_handler_closes_span( + wired_otel, server_span_factory, code, path +): + request = _fake_request(parent_otel_span=server_span_factory(path)) + exc = ProxyException( + message="boom", + type="invalid_request_error", + param="request_body", + code=code, + ) + response = asyncio.run(openai_exception_handler(request, exc)) + assert response.status_code == code + assert_server_span_attrs( + wired_otel, + expected_status=code, + expected_url_path=path, + where=f"openai_exception_handler ({path} code={code})", + ) + assert request.state.parent_otel_span is None diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_passthrough_endpoints.py b/tests/test_litellm/integrations/open_telemetry/test_otel_passthrough_endpoints.py new file mode 100644 index 000000000000..4b24b6c487d1 --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_passthrough_endpoints.py @@ -0,0 +1,136 @@ +"""LIT-3193 — passthrough endpoints. Drives proxy_logging.post_call_failure_hook +(the integration point pass_through_endpoint reaches on upstream >=300).""" + +import asyncio + +import pytest +from fastapi import HTTPException + +from litellm.caching.dual_cache import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import ProxyLogging + +from ._helpers import ( + assert_server_span_attrs, + make_fastapi_http_exception, + make_httpx_status_error, +) + + +def _real_user_api_key_dict(parent_span): + return UserAPIKeyAuth( + api_key="sk-test", + team_id="team-lit-3193", + team_alias="lit-3193-team", + parent_otel_span=parent_span, + ) + + +def _proxy_logging(): + return ProxyLogging(user_api_key_cache=UserApiKeyCache(DualCache())) + + +def _drive_passthrough_failure(*, exception, user_api_key_dict): + asyncio.run( + _proxy_logging().post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=exception, + request_data={}, + ) + ) + + +VERTEX_PATH = "/vertex_ai/v1/projects/p/locations/us-central1/publishers/google/models/gemini-1.5-pro:generateContent" + + +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(401, "no proxy key"), 401), + (make_fastapi_http_exception(400, "bad request"), 400), + (make_fastapi_http_exception(403, "upstream forbidden"), 403), + (make_fastapi_http_exception(404, "upstream not found"), 404), + (make_fastapi_http_exception(429, "upstream rate limit"), 429), + (make_httpx_status_error(500, "upstream blew up"), 500), + (make_httpx_status_error(502, "bad gateway"), 502), + (make_httpx_status_error(503, "service unavailable"), 503), + (make_fastapi_http_exception(502, "wrapped 502"), 502), + ], + ids=[ + "401-litellm-auth", + "400-upstream", + "403-upstream", + "404-upstream", + "429-upstream", + "500-upstream-httpx", + "502-upstream-httpx", + "503-upstream-httpx", + "502-wrapped", + ], +) +def test_vertex_passthrough_failure_stamps_server_span( + exception, + expected_status, + server_span_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory( + VERTEX_PATH, http_route="/vertex_ai/{endpoint:path}" + ) + uakd = _real_user_api_key_dict(server_span) + + _drive_passthrough_failure(exception=exception, user_api_key_dict=uakd) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=VERTEX_PATH, + expected_http_route="/vertex_ai/{endpoint:path}", + where=f"vertex passthrough {expected_status}", + ) + + +SMOKE_PASSTHROUGHS = [ + ("/bedrock/model/anthropic.claude-v2/invoke", "/bedrock/{endpoint:path}"), + ("/anthropic/v1/messages", "/anthropic/{endpoint:path}"), + ("/openai/v1/chat/completions", "/openai/{endpoint:path}"), + ("/gemini/v1beta/models/gemini-pro:generateContent", "/gemini/{endpoint:path}"), + ("/cohere/v1/chat", "/cohere/{endpoint:path}"), + ("/azure/openai/deployments/gpt4/chat/completions", "/azure/{endpoint:path}"), +] + + +@pytest.mark.parametrize("path,http_route", SMOKE_PASSTHROUGHS) +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(400, "upstream bad request"), 400), + (make_httpx_status_error(502, "upstream"), 502), + ], + ids=["400", "502"], +) +def test_passthrough_failure_stamps_server_span( + path, + http_route, + exception, + expected_status, + server_span_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory(path, http_route=http_route) + uakd = _real_user_api_key_dict(server_span) + + _drive_passthrough_failure(exception=exception, user_api_key_dict=uakd) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=path, + expected_http_route=http_route, + where=f"{path} {expected_status}", + ) diff --git a/tests/test_litellm/integrations/open_telemetry/test_otel_unified_endpoints.py b/tests/test_litellm/integrations/open_telemetry/test_otel_unified_endpoints.py new file mode 100644 index 000000000000..0279c22154ed --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/test_otel_unified_endpoints.py @@ -0,0 +1,223 @@ +"""LIT-3193 — unified inference endpoints. Drives _handle_llm_api_exception +to assert SERVER-span attrs (status, url.path, http.route, duration).""" + +import asyncio + +import pytest +from fastapi import HTTPException + +from opentelemetry.trace import Status, StatusCode + +import litellm +from litellm.caching.dual_cache import DualCache +from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import ProxyLogging + +from ._helpers import ( + HttpStatusException, + assert_server_span_attrs, + make_fastapi_http_exception, + make_httpx_status_error, +) + + +def _real_user_api_key_dict(parent_span): + return UserAPIKeyAuth( + api_key="sk-test", + team_id="team-lit-3193", + team_alias="lit-3193-team", + parent_otel_span=parent_span, + ) + + +def _proxy_logging(): + return ProxyLogging(user_api_key_cache=UserApiKeyCache(DualCache())) + + +def _drive_unified_failure( + *, + exception, + server_span, + user_api_key_dict, +): + proc = ProxyBaseLLMRequestProcessing(data={}) + try: + asyncio.run( + proc._handle_llm_api_exception( + e=exception, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=_proxy_logging(), + ) + ) + except (ProxyException, HTTPException): + pass + + +CHAT_PATH = "/v1/chat/completions" + + +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(400, "bad request"), 400), + (make_fastapi_http_exception(401, "no key"), 401), + (make_fastapi_http_exception(403, "no model access"), 403), + (make_fastapi_http_exception(404, "model not in router"), 404), + (make_fastapi_http_exception(422, "validation"), 422), + (make_fastapi_http_exception(429, "rate limit"), 429), + (HttpStatusException(500, "uncaught"), 500), + (make_httpx_status_error(502, "upstream blew up"), 502), + (make_httpx_status_error(503, "upstream down"), 503), + (make_httpx_status_error(504, "upstream timeout"), 504), + ], + ids=[ + "400-bad-request", + "401-no-key", + "403-no-model-access", + "404-model-not-found", + "422-validation", + "429-rate-limit", + "500-uncaught", + "502-upstream", + "503-upstream", + "504-upstream-timeout", + ], +) +def test_chat_completions_failure_stamps_server_span( + exception, + expected_status, + server_span_factory, + user_api_key_dict_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory(CHAT_PATH) + uakd = _real_user_api_key_dict(server_span) + + _drive_unified_failure( + exception=exception, server_span=server_span, user_api_key_dict=uakd + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=CHAT_PATH, + where=f"chat/completions {expected_status}", + ) + + +def test_chat_completions_success_path_stamps_200( + otel_with_exporter, server_span_factory +): + otel, exporter = otel_with_exporter + server_span = server_span_factory(CHAT_PATH) + _real_user_api_key_dict(server_span) + + otel.set_response_status_code_attribute(server_span, 200) + otel.set_preprocessing_duration_attribute(server_span, {}) + server_span.set_status(Status(StatusCode.OK)) + server_span.end() + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=CHAT_PATH, + where="chat/completions 200", + ) + + +# /v1/responses ends the proxy span before async_post_call_success_hook fires, +# so the 200 stamp must happen at span close (here), not in the hook. +@pytest.mark.parametrize( + "path", ["/v1/chat/completions", "/v1/messages", "/v1/responses"] +) +def test_end_proxy_span_from_kwargs_stamps_200( + path, otel_with_exporter, server_span_factory +): + from datetime import datetime + + otel, exporter = otel_with_exporter + server_span = server_span_factory(path) + kwargs = {"litellm_params": {"metadata": {"litellm_parent_otel_span": server_span}}} + otel._end_proxy_span_from_kwargs(kwargs, datetime.now()) + + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=path, + where=f"{path} _end_proxy_span_from_kwargs", + ) + + +# Bare TypeError has no .code/.status_code, so error_information.error_code is +# empty and _record_exception_on_span skips the stamp — must default to 500. +def test_async_post_call_failure_hook_defaults_to_500( + otel_with_exporter, server_span_factory +): + otel, exporter = otel_with_exporter + server_span = server_span_factory("/v1/responses") + uakd = _real_user_api_key_dict(server_span) + + asyncio.run( + otel.async_post_call_failure_hook( + request_data={}, + original_exception=TypeError("missing required argument"), + user_api_key_dict=uakd, + ) + ) + + assert_server_span_attrs( + exporter, + expected_status=500, + expected_url_path="/v1/responses", + where="async_post_call_failure_hook (TypeError) defaults to 500", + ) + + +SMOKE_ENDPOINTS = [ + "/v1/embeddings", + "/v1/completions", + "/v1/images/generations", + "/v1/audio/speech", + "/v1/audio/transcriptions", + "/v1/moderations", + "/v1/rerank", + "/v1/responses", + "/v1/messages", +] + + +@pytest.mark.parametrize("path", SMOKE_ENDPOINTS) +@pytest.mark.parametrize( + "exception, expected_status", + [ + (make_fastapi_http_exception(401, "no key"), 401), + (make_httpx_status_error(502, "upstream"), 502), + ], + ids=["401", "502"], +) +def test_unified_endpoint_failure_stamps_server_span( + path, + exception, + expected_status, + server_span_factory, + otel_with_exporter, + register_otel_callback, +): + _otel, exporter = otel_with_exporter + server_span = server_span_factory(path) + uakd = _real_user_api_key_dict(server_span) + + _drive_unified_failure( + exception=exception, server_span=server_span, user_api_key_dict=uakd + ) + + assert_server_span_attrs( + exporter, + expected_status=expected_status, + expected_url_path=path, + where=f"{path} {expected_status}", + ) From 8513d7fc0c3bdc44ece318870c65029739cbf31b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 23 May 2026 13:05:26 -0700 Subject: [PATCH 008/109] chore: update Next.js build artifacts (2026-05-23 19:21 UTC, node v20.20.2) (#28707) --- .../out/{404/index.html => 404.html} | 2 +- .../_experimental/out/__next.__PAGE__.txt | 26 +- .../proxy/_experimental/out/__next._full.txt | 44 ++-- .../proxy/_experimental/out/__next._head.txt | 2 +- .../proxy/_experimental/out/__next._index.txt | 2 +- .../proxy/_experimental/out/__next._tree.txt | 2 +- ...c0e9dc19dbbd4ed.js => 003f1ffc5817ab83.js} | 2 +- .../_next/static/chunks/048f065ef4eab631.js | 1 + .../_next/static/chunks/06b2ea8c776c2e9b.js | 1 - .../_next/static/chunks/0974abc09c5e7ada.js | 1 + ...30fbd1b8bfd0d5f.js => 1251d58bd3ba113b.js} | 4 +- ...a1c7742cc6fe8b4.js => 16a1651c0b3e7c8e.js} | 18 +- .../_next/static/chunks/16c0e58809eaf2b5.js | 72 ++++++ ...d02f158353d9cea.js => 1c881baaaa68b7a5.js} | 4 +- .../_next/static/chunks/23bfdf9b0544f0b1.js | 10 + .../_next/static/chunks/325c5effcc6e2384.js | 1 - ...492aee8924914ae.js => 3310f8d28e1d8cfa.js} | 2 +- .../_next/static/chunks/35c3d528354208f4.js | 1 + .../_next/static/chunks/43118e9b10d4adb7.js | 1 - .../_next/static/chunks/4a0ccb5ed3d0c33f.js | 72 ------ .../_next/static/chunks/4d3aafceafce4648.js | 1 - ...6d6c6bf01dbe363.js => 4e3eafbea2035508.js} | 4 +- .../_next/static/chunks/4eab0601ddda6611.js | 3 - .../_next/static/chunks/5181a28310842d3d.js | 3 + .../_next/static/chunks/5583bc893837fdf8.js | 4 - .../_next/static/chunks/561b33e0e4675dd8.js | 1 - ...8044d049f37fcb0.js => 5a69756708c8900c.js} | 2 +- .../_next/static/chunks/5c9bf87d25400872.js | 1 + .../_next/static/chunks/60d3701e4f82c4ff.js | 1 + .../_next/static/chunks/6188170a32c9a3c3.js | 3 - .../_next/static/chunks/631b1874cba557c9.js | 91 +++++++ .../_next/static/chunks/667115385c71d974.js | 1 - ...7919217b70a9a35.js => 6df5e16ab3d367ef.js} | 18 +- .../_next/static/chunks/6edd697afbcf3405.js | 1 + .../_next/static/chunks/7142b7d05b1b4b2e.js | 1 - ...0acf4fa815c638e.js => 726bebeef472c6cb.js} | 2 +- .../_next/static/chunks/7b6bca6d63438103.js | 1 + .../_next/static/chunks/7cfedb9cfc78efdb.js | 1 - ...55b9871de1cf382.js => 80fc3fb8d0c44655.js} | 2 +- .../_next/static/chunks/827c38ee3538aeb8.js | 1 + .../_next/static/chunks/8712a14511b63b6c.js | 1 - ...66fce1df297fb45.js => 87573aaa9c57fc3a.js} | 4 +- .../_next/static/chunks/88001a7ecaf7b1af.js | 1 + ...6ab357d1bbb53f0.js => 8f3bf592254c6c3b.js} | 2 +- .../_next/static/chunks/934dbc43f8c1abde.js | 10 - .../_next/static/chunks/94f7208f5087e27c.js | 1 - .../_next/static/chunks/9955c118354ef6cc.js | 231 ++++++++++++++++++ .../_next/static/chunks/a2352cfbbb384fe9.js | 1 - .../_next/static/chunks/aaa545ba3e90f434.js | 1 + ...19edc2f5c8ae8a5.js => ac3eece174ae3ee9.js} | 4 +- ...0dc4591ef08a91f.js => ae625aa52246581e.js} | 2 +- .../_next/static/chunks/af6fc0727c3097de.js | 1 + ...20b47ad353e1fc9.js => b4505a784b9b23e6.js} | 4 +- .../_next/static/chunks/b6c1a99750c8786e.js | 1 - ...7bc6bc3b0d44586.js => bd31e2f87615de8b.js} | 2 +- ...606c92ecd600e0c.js => bd799dc9aa7f786a.js} | 2 +- ...df02e15c80921ac.js => c439a1e9093448b5.js} | 2 +- .../_next/static/chunks/c7db6d1325b26f45.js | 1 + .../_next/static/chunks/ca7a3fdb635fb7dc.js | 231 ------------------ ...3f6fc3c2ab9cf23.js => cbc99c8fae110c02.js} | 6 +- .../_next/static/chunks/d3d0acca9a72b37a.js | 1 + ...3520f5ea04a66c8.js => d720c3179e45c754.js} | 30 +-- .../_next/static/chunks/dbf6a58fdc648c8d.js | 1 + .../_next/static/chunks/e00ff4e948e36ad2.js | 1 - .../_next/static/chunks/e1a670efcb966aaa.js | 91 ------- ...81db9a7d7d3325e.js => e29e363f6c8abbd7.js} | 4 +- ...e1da84ff36bc348.js => e620284e1d071312.js} | 2 +- .../_next/static/chunks/e7e5bfdf70ba79ab.js | 1 - .../_next/static/chunks/eaeb6c071ee29f14.js | 3 + .../_next/static/chunks/ecbe4d32f0d0ccd3.js | 1 - .../_next/static/chunks/f5de144852609bdf.js | 1 - ...4f351822ff5289f.js => f6fe773610e02694.js} | 2 +- ...46b8595ea98bdef.js => f739f683b0e3528d.js} | 10 +- ...5bdda1c2974030d.js => fa11bea8d4771df2.js} | 2 +- .../_next/static/chunks/fc5911e3a3caa8aa.js | 1 + .../_buildManifest.js | 0 .../_clientMiddlewareManifest.json | 0 .../_ssgManifest.js | 0 .../index.html => _not-found.html} | 2 +- .../proxy/_experimental/out/_not-found.txt | 2 +- .../out/_not-found/__next._full.txt | 2 +- .../out/_not-found/__next._head.txt | 2 +- .../out/_not-found/__next._index.txt | 2 +- .../_not-found/__next._not-found.__PAGE__.txt | 2 +- .../out/_not-found/__next._not-found.txt | 2 +- .../out/_not-found/__next._tree.txt | 2 +- .../index.html => api-reference.html} | 2 +- .../proxy/_experimental/out/api-reference.txt | 6 +- ...KGRhc2hib2FyZCk.api-reference.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.api-reference.txt | 2 +- .../api-reference/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/api-reference/__next._full.txt | 6 +- .../out/api-reference/__next._head.txt | 2 +- .../out/api-reference/__next._index.txt | 2 +- .../out/api-reference/__next._tree.txt | 2 +- .../out/{chat/index.html => chat.html} | 2 +- litellm/proxy/_experimental/out/chat.txt | 4 +- .../_experimental/out/chat/__next._full.txt | 4 +- .../_experimental/out/chat/__next._head.txt | 2 +- .../_experimental/out/chat/__next._index.txt | 2 +- .../_experimental/out/chat/__next._tree.txt | 2 +- .../out/chat/__next.chat.__PAGE__.txt | 4 +- .../_experimental/out/chat/__next.chat.txt | 2 +- .../index.html => api-playground.html} | 2 +- .../out/experimental/api-playground.txt | 6 +- ...k.experimental.api-playground.__PAGE__.txt | 4 +- ...2hib2FyZCk.experimental.api-playground.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.experimental.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.txt | 4 +- .../api-playground/__next._full.txt | 6 +- .../api-playground/__next._head.txt | 2 +- .../api-playground/__next._index.txt | 2 +- .../api-playground/__next._tree.txt | 2 +- .../{budgets/index.html => budgets.html} | 2 +- .../out/experimental/budgets.txt | 6 +- ...ib2FyZCk.experimental.budgets.__PAGE__.txt | 4 +- ....!KGRhc2hib2FyZCk.experimental.budgets.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.experimental.txt | 2 +- .../budgets/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/experimental/budgets/__next._full.txt | 6 +- .../out/experimental/budgets/__next._head.txt | 2 +- .../experimental/budgets/__next._index.txt | 2 +- .../out/experimental/budgets/__next._tree.txt | 2 +- .../{caching/index.html => caching.html} | 2 +- .../out/experimental/caching.txt | 8 +- ...ib2FyZCk.experimental.caching.__PAGE__.txt | 4 +- ....!KGRhc2hib2FyZCk.experimental.caching.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.experimental.txt | 2 +- .../caching/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/experimental/caching/__next._full.txt | 8 +- .../out/experimental/caching/__next._head.txt | 2 +- .../experimental/caching/__next._index.txt | 2 +- .../out/experimental/caching/__next._tree.txt | 2 +- .../index.html => claude-code-plugins.html} | 2 +- .../out/experimental/claude-code-plugins.txt | 6 +- ...erimental.claude-code-plugins.__PAGE__.txt | 4 +- ...FyZCk.experimental.claude-code-plugins.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.experimental.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.txt | 4 +- .../claude-code-plugins/__next._full.txt | 6 +- .../claude-code-plugins/__next._head.txt | 2 +- .../claude-code-plugins/__next._index.txt | 2 +- .../claude-code-plugins/__next._tree.txt | 2 +- .../{old-usage/index.html => old-usage.html} | 2 +- .../out/experimental/old-usage.txt | 8 +- ...2FyZCk.experimental.old-usage.__PAGE__.txt | 4 +- ...KGRhc2hib2FyZCk.experimental.old-usage.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.experimental.txt | 2 +- .../old-usage/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../experimental/old-usage/__next._full.txt | 8 +- .../experimental/old-usage/__next._head.txt | 2 +- .../experimental/old-usage/__next._index.txt | 2 +- .../experimental/old-usage/__next._tree.txt | 2 +- .../{prompts/index.html => prompts.html} | 2 +- .../out/experimental/prompts.txt | 6 +- ...ib2FyZCk.experimental.prompts.__PAGE__.txt | 4 +- ....!KGRhc2hib2FyZCk.experimental.prompts.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.experimental.txt | 2 +- .../prompts/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/experimental/prompts/__next._full.txt | 6 +- .../out/experimental/prompts/__next._head.txt | 2 +- .../experimental/prompts/__next._index.txt | 2 +- .../out/experimental/prompts/__next._tree.txt | 2 +- .../index.html => tag-management.html} | 2 +- .../out/experimental/tag-management.txt | 6 +- ...k.experimental.tag-management.__PAGE__.txt | 4 +- ...2hib2FyZCk.experimental.tag-management.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.experimental.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.txt | 4 +- .../tag-management/__next._full.txt | 6 +- .../tag-management/__next._head.txt | 2 +- .../tag-management/__next._index.txt | 2 +- .../tag-management/__next._tree.txt | 2 +- .../index.html => guardrails.html} | 2 +- .../proxy/_experimental/out/guardrails.txt | 6 +- ...t.!KGRhc2hib2FyZCk.guardrails.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.guardrails.txt | 2 +- .../guardrails/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/guardrails/__next._full.txt | 6 +- .../out/guardrails/__next._head.txt | 2 +- .../out/guardrails/__next._index.txt | 2 +- .../out/guardrails/__next._tree.txt | 2 +- litellm/proxy/_experimental/out/index.html | 2 +- litellm/proxy/_experimental/out/index.txt | 44 ++-- .../out/{login/index.html => login.html} | 2 +- litellm/proxy/_experimental/out/login.txt | 4 +- .../_experimental/out/login/__next._full.txt | 4 +- .../_experimental/out/login/__next._head.txt | 2 +- .../_experimental/out/login/__next._index.txt | 2 +- .../_experimental/out/login/__next._tree.txt | 2 +- .../out/login/__next.login.__PAGE__.txt | 4 +- .../_experimental/out/login/__next.login.txt | 2 +- .../out/{logs/index.html => logs.html} | 2 +- litellm/proxy/_experimental/out/logs.txt | 8 +- .../__next.!KGRhc2hib2FyZCk.logs.__PAGE__.txt | 4 +- .../out/logs/__next.!KGRhc2hib2FyZCk.logs.txt | 2 +- .../out/logs/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../_experimental/out/logs/__next._full.txt | 8 +- .../_experimental/out/logs/__next._head.txt | 2 +- .../_experimental/out/logs/__next._index.txt | 2 +- .../_experimental/out/logs/__next._tree.txt | 2 +- .../{callback/index.html => callback.html} | 2 +- .../_experimental/out/mcp/oauth/callback.txt | 4 +- .../out/mcp/oauth/callback/__next._full.txt | 4 +- .../out/mcp/oauth/callback/__next._head.txt | 2 +- .../out/mcp/oauth/callback/__next._index.txt | 2 +- .../out/mcp/oauth/callback/__next._tree.txt | 2 +- .../__next.mcp.oauth.callback.__PAGE__.txt | 4 +- .../callback/__next.mcp.oauth.callback.txt | 2 +- .../mcp/oauth/callback/__next.mcp.oauth.txt | 2 +- .../out/mcp/oauth/callback/__next.mcp.txt | 2 +- .../{model-hub/index.html => model-hub.html} | 2 +- litellm/proxy/_experimental/out/model-hub.txt | 8 +- ...xt.!KGRhc2hib2FyZCk.model-hub.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.model-hub.txt | 2 +- .../out/model-hub/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/model-hub/__next._full.txt | 8 +- .../out/model-hub/__next._head.txt | 2 +- .../out/model-hub/__next._index.txt | 2 +- .../out/model-hub/__next._tree.txt | 2 +- .../{model_hub/index.html => model_hub.html} | 2 +- litellm/proxy/_experimental/out/model_hub.txt | 4 +- .../out/model_hub/__next._full.txt | 4 +- .../out/model_hub/__next._head.txt | 2 +- .../out/model_hub/__next._index.txt | 2 +- .../out/model_hub/__next._tree.txt | 2 +- .../model_hub/__next.model_hub.__PAGE__.txt | 4 +- .../out/model_hub/__next.model_hub.txt | 2 +- .../index.html => model_hub_table.html} | 2 +- .../_experimental/out/model_hub_table.txt | 6 +- .../out/model_hub_table/__next._full.txt | 6 +- .../out/model_hub_table/__next._head.txt | 2 +- .../out/model_hub_table/__next._index.txt | 2 +- .../out/model_hub_table/__next._tree.txt | 2 +- .../__next.model_hub_table.__PAGE__.txt | 4 +- .../__next.model_hub_table.txt | 2 +- .../index.html => models-and-endpoints.html} | 2 +- .../out/models-and-endpoints.txt | 8 +- ...ib2FyZCk.models-and-endpoints.__PAGE__.txt | 4 +- ....!KGRhc2hib2FyZCk.models-and-endpoints.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/models-and-endpoints/__next._full.txt | 8 +- .../out/models-and-endpoints/__next._head.txt | 2 +- .../models-and-endpoints/__next._index.txt | 2 +- .../out/models-and-endpoints/__next._tree.txt | 2 +- .../index.html => onboarding.html} | 2 +- .../proxy/_experimental/out/onboarding.txt | 4 +- .../out/onboarding/__next._full.txt | 4 +- .../out/onboarding/__next._head.txt | 2 +- .../out/onboarding/__next._index.txt | 2 +- .../out/onboarding/__next._tree.txt | 2 +- .../onboarding/__next.onboarding.__PAGE__.txt | 4 +- .../out/onboarding/__next.onboarding.txt | 2 +- .../index.html => organizations.html} | 2 +- .../proxy/_experimental/out/organizations.txt | 6 +- ...KGRhc2hib2FyZCk.organizations.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.organizations.txt | 2 +- .../organizations/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/organizations/__next._full.txt | 6 +- .../out/organizations/__next._head.txt | 2 +- .../out/organizations/__next._index.txt | 2 +- .../out/organizations/__next._tree.txt | 2 +- .../index.html => playground.html} | 2 +- .../proxy/_experimental/out/playground.txt | 8 +- ...t.!KGRhc2hib2FyZCk.playground.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.playground.txt | 2 +- .../playground/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/playground/__next._full.txt | 8 +- .../out/playground/__next._head.txt | 2 +- .../out/playground/__next._index.txt | 2 +- .../out/playground/__next._tree.txt | 2 +- .../{policies/index.html => policies.html} | 2 +- litellm/proxy/_experimental/out/policies.txt | 8 +- ...ext.!KGRhc2hib2FyZCk.policies.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.policies.txt | 2 +- .../out/policies/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/policies/__next._full.txt | 8 +- .../out/policies/__next._head.txt | 2 +- .../out/policies/__next._index.txt | 2 +- .../out/policies/__next._tree.txt | 2 +- .../index.html => admin-settings.html} | 2 +- .../out/settings/admin-settings.txt | 6 +- ...FyZCk.settings.admin-settings.__PAGE__.txt | 4 +- ...GRhc2hib2FyZCk.settings.admin-settings.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.settings.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.txt | 4 +- .../settings/admin-settings/__next._full.txt | 6 +- .../settings/admin-settings/__next._head.txt | 2 +- .../settings/admin-settings/__next._index.txt | 2 +- .../settings/admin-settings/__next._tree.txt | 2 +- .../index.html => logging-and-alerts.html} | 2 +- .../out/settings/logging-and-alerts.txt | 8 +- ...k.settings.logging-and-alerts.__PAGE__.txt | 4 +- ...2hib2FyZCk.settings.logging-and-alerts.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.settings.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.txt | 4 +- .../logging-and-alerts/__next._full.txt | 8 +- .../logging-and-alerts/__next._head.txt | 2 +- .../logging-and-alerts/__next._index.txt | 2 +- .../logging-and-alerts/__next._tree.txt | 2 +- .../index.html => router-settings.html} | 2 +- .../out/settings/router-settings.txt | 8 +- ...yZCk.settings.router-settings.__PAGE__.txt | 4 +- ...Rhc2hib2FyZCk.settings.router-settings.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.settings.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.txt | 4 +- .../settings/router-settings/__next._full.txt | 8 +- .../settings/router-settings/__next._head.txt | 2 +- .../router-settings/__next._index.txt | 2 +- .../settings/router-settings/__next._tree.txt | 2 +- .../{ui-theme/index.html => ui-theme.html} | 2 +- .../_experimental/out/settings/ui-theme.txt | 6 +- .../__next.!KGRhc2hib2FyZCk.settings.txt | 2 +- ...c2hib2FyZCk.settings.ui-theme.__PAGE__.txt | 4 +- ...ext.!KGRhc2hib2FyZCk.settings.ui-theme.txt | 2 +- .../ui-theme/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/settings/ui-theme/__next._full.txt | 6 +- .../out/settings/ui-theme/__next._head.txt | 2 +- .../out/settings/ui-theme/__next._index.txt | 2 +- .../out/settings/ui-theme/__next._tree.txt | 2 +- .../out/{skills/index.html => skills.html} | 2 +- litellm/proxy/_experimental/out/skills.txt | 6 +- ..._next.!KGRhc2hib2FyZCk.skills.__PAGE__.txt | 4 +- .../skills/__next.!KGRhc2hib2FyZCk.skills.txt | 2 +- .../out/skills/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../_experimental/out/skills/__next._full.txt | 6 +- .../_experimental/out/skills/__next._head.txt | 2 +- .../out/skills/__next._index.txt | 2 +- .../_experimental/out/skills/__next._tree.txt | 2 +- .../out/{teams/index.html => teams.html} | 2 +- litellm/proxy/_experimental/out/teams.txt | 8 +- ...__next.!KGRhc2hib2FyZCk.teams.__PAGE__.txt | 4 +- .../teams/__next.!KGRhc2hib2FyZCk.teams.txt | 2 +- .../out/teams/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../_experimental/out/teams/__next._full.txt | 8 +- .../_experimental/out/teams/__next._head.txt | 2 +- .../_experimental/out/teams/__next._index.txt | 2 +- .../_experimental/out/teams/__next._tree.txt | 2 +- .../{test-key/index.html => test-key.html} | 2 +- litellm/proxy/_experimental/out/test-key.txt | 8 +- ...ext.!KGRhc2hib2FyZCk.test-key.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.test-key.txt | 2 +- .../out/test-key/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/test-key/__next._full.txt | 8 +- .../out/test-key/__next._head.txt | 2 +- .../out/test-key/__next._index.txt | 2 +- .../out/test-key/__next._tree.txt | 2 +- .../index.html => mcp-servers.html} | 2 +- .../_experimental/out/tools/mcp-servers.txt | 8 +- ...c2hib2FyZCk.tools.mcp-servers.__PAGE__.txt | 4 +- ...ext.!KGRhc2hib2FyZCk.tools.mcp-servers.txt | 2 +- .../__next.!KGRhc2hib2FyZCk.tools.txt | 2 +- .../mcp-servers/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/tools/mcp-servers/__next._full.txt | 8 +- .../out/tools/mcp-servers/__next._head.txt | 2 +- .../out/tools/mcp-servers/__next._index.txt | 2 +- .../out/tools/mcp-servers/__next._tree.txt | 2 +- .../index.html => vector-stores.html} | 2 +- .../_experimental/out/tools/vector-stores.txt | 6 +- .../__next.!KGRhc2hib2FyZCk.tools.txt | 2 +- ...hib2FyZCk.tools.vector-stores.__PAGE__.txt | 4 +- ...t.!KGRhc2hib2FyZCk.tools.vector-stores.txt | 2 +- .../vector-stores/__next.!KGRhc2hib2FyZCk.txt | 4 +- .../out/tools/vector-stores/__next._full.txt | 6 +- .../out/tools/vector-stores/__next._head.txt | 2 +- .../out/tools/vector-stores/__next._index.txt | 2 +- .../out/tools/vector-stores/__next._tree.txt | 2 +- .../out/{usage/index.html => usage.html} | 2 +- litellm/proxy/_experimental/out/usage.txt | 8 +- .../out/usage/__next.!KGRhc2hib2FyZCk.txt | 4 +- ...__next.!KGRhc2hib2FyZCk.usage.__PAGE__.txt | 4 +- .../usage/__next.!KGRhc2hib2FyZCk.usage.txt | 2 +- .../_experimental/out/usage/__next._full.txt | 8 +- .../_experimental/out/usage/__next._head.txt | 2 +- .../_experimental/out/usage/__next._index.txt | 2 +- .../_experimental/out/usage/__next._tree.txt | 2 +- .../out/{users/index.html => users.html} | 2 +- litellm/proxy/_experimental/out/users.txt | 6 +- .../out/users/__next.!KGRhc2hib2FyZCk.txt | 4 +- ...__next.!KGRhc2hib2FyZCk.users.__PAGE__.txt | 4 +- .../users/__next.!KGRhc2hib2FyZCk.users.txt | 2 +- .../_experimental/out/users/__next._full.txt | 6 +- .../_experimental/out/users/__next._head.txt | 2 +- .../_experimental/out/users/__next._index.txt | 2 +- .../_experimental/out/users/__next._tree.txt | 2 +- .../index.html => virtual-keys.html} | 2 +- .../proxy/_experimental/out/virtual-keys.txt | 6 +- .../virtual-keys/__next.!KGRhc2hib2FyZCk.txt | 4 +- ...!KGRhc2hib2FyZCk.virtual-keys.__PAGE__.txt | 4 +- .../__next.!KGRhc2hib2FyZCk.virtual-keys.txt | 2 +- .../out/virtual-keys/__next._full.txt | 6 +- .../out/virtual-keys/__next._head.txt | 2 +- .../out/virtual-keys/__next._index.txt | 2 +- .../out/virtual-keys/__next._tree.txt | 2 +- 394 files changed, 1077 insertions(+), 1082 deletions(-) rename litellm/proxy/_experimental/out/{404/index.html => 404.html} (98%) rename litellm/proxy/_experimental/out/_next/static/chunks/{3c0e9dc19dbbd4ed.js => 003f1ffc5817ab83.js} (55%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/048f065ef4eab631.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/06b2ea8c776c2e9b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0974abc09c5e7ada.js rename litellm/proxy/_experimental/out/_next/static/chunks/{030fbd1b8bfd0d5f.js => 1251d58bd3ba113b.js} (89%) rename litellm/proxy/_experimental/out/_next/static/chunks/{da1c7742cc6fe8b4.js => 16a1651c0b3e7c8e.js} (54%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/16c0e58809eaf2b5.js rename litellm/proxy/_experimental/out/_next/static/chunks/{bd02f158353d9cea.js => 1c881baaaa68b7a5.js} (81%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/23bfdf9b0544f0b1.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/325c5effcc6e2384.js rename litellm/proxy/_experimental/out/_next/static/chunks/{9492aee8924914ae.js => 3310f8d28e1d8cfa.js} (91%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/35c3d528354208f4.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/43118e9b10d4adb7.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4a0ccb5ed3d0c33f.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4d3aafceafce4648.js rename litellm/proxy/_experimental/out/_next/static/chunks/{26d6c6bf01dbe363.js => 4e3eafbea2035508.js} (81%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4eab0601ddda6611.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5181a28310842d3d.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5583bc893837fdf8.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/561b33e0e4675dd8.js rename litellm/proxy/_experimental/out/_next/static/chunks/{e8044d049f37fcb0.js => 5a69756708c8900c.js} (94%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5c9bf87d25400872.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/60d3701e4f82c4ff.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6188170a32c9a3c3.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/631b1874cba557c9.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/667115385c71d974.js rename litellm/proxy/_experimental/out/_next/static/chunks/{a7919217b70a9a35.js => 6df5e16ab3d367ef.js} (54%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6edd697afbcf3405.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7142b7d05b1b4b2e.js rename litellm/proxy/_experimental/out/_next/static/chunks/{20acf4fa815c638e.js => 726bebeef472c6cb.js} (94%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7b6bca6d63438103.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7cfedb9cfc78efdb.js rename litellm/proxy/_experimental/out/_next/static/chunks/{355b9871de1cf382.js => 80fc3fb8d0c44655.js} (51%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/827c38ee3538aeb8.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8712a14511b63b6c.js rename litellm/proxy/_experimental/out/_next/static/chunks/{266fce1df297fb45.js => 87573aaa9c57fc3a.js} (82%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/88001a7ecaf7b1af.js rename litellm/proxy/_experimental/out/_next/static/chunks/{d6ab357d1bbb53f0.js => 8f3bf592254c6c3b.js} (99%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/934dbc43f8c1abde.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/94f7208f5087e27c.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9955c118354ef6cc.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a2352cfbbb384fe9.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/aaa545ba3e90f434.js rename litellm/proxy/_experimental/out/_next/static/chunks/{d19edc2f5c8ae8a5.js => ac3eece174ae3ee9.js} (81%) rename litellm/proxy/_experimental/out/_next/static/chunks/{10dc4591ef08a91f.js => ae625aa52246581e.js} (98%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/af6fc0727c3097de.js rename litellm/proxy/_experimental/out/_next/static/chunks/{120b47ad353e1fc9.js => b4505a784b9b23e6.js} (83%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/b6c1a99750c8786e.js rename litellm/proxy/_experimental/out/_next/static/chunks/{d7bc6bc3b0d44586.js => bd31e2f87615de8b.js} (86%) rename litellm/proxy/_experimental/out/_next/static/chunks/{0606c92ecd600e0c.js => bd799dc9aa7f786a.js} (96%) rename litellm/proxy/_experimental/out/_next/static/chunks/{edf02e15c80921ac.js => c439a1e9093448b5.js} (92%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c7db6d1325b26f45.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ca7a3fdb635fb7dc.js rename litellm/proxy/_experimental/out/_next/static/chunks/{43f6fc3c2ab9cf23.js => cbc99c8fae110c02.js} (63%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d3d0acca9a72b37a.js rename litellm/proxy/_experimental/out/_next/static/chunks/{23520f5ea04a66c8.js => d720c3179e45c754.js} (94%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/dbf6a58fdc648c8d.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e00ff4e948e36ad2.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e1a670efcb966aaa.js rename litellm/proxy/_experimental/out/_next/static/chunks/{a81db9a7d7d3325e.js => e29e363f6c8abbd7.js} (67%) rename litellm/proxy/_experimental/out/_next/static/chunks/{1e1da84ff36bc348.js => e620284e1d071312.js} (55%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e7e5bfdf70ba79ab.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/eaeb6c071ee29f14.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ecbe4d32f0d0ccd3.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/f5de144852609bdf.js rename litellm/proxy/_experimental/out/_next/static/chunks/{84f351822ff5289f.js => f6fe773610e02694.js} (97%) rename litellm/proxy/_experimental/out/_next/static/chunks/{446b8595ea98bdef.js => f739f683b0e3528d.js} (75%) rename litellm/proxy/_experimental/out/_next/static/chunks/{95bdda1c2974030d.js => fa11bea8d4771df2.js} (97%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/fc5911e3a3caa8aa.js rename litellm/proxy/_experimental/out/_next/static/{LpD6ruZoEpvYpT5IvMEoa => wnL6e5S6xaG1UdkxtYrTo}/_buildManifest.js (100%) rename litellm/proxy/_experimental/out/_next/static/{LpD6ruZoEpvYpT5IvMEoa => wnL6e5S6xaG1UdkxtYrTo}/_clientMiddlewareManifest.json (100%) rename litellm/proxy/_experimental/out/_next/static/{LpD6ruZoEpvYpT5IvMEoa => wnL6e5S6xaG1UdkxtYrTo}/_ssgManifest.js (100%) rename litellm/proxy/_experimental/out/{_not-found/index.html => _not-found.html} (98%) rename litellm/proxy/_experimental/out/{api-reference/index.html => api-reference.html} (97%) rename litellm/proxy/_experimental/out/{chat/index.html => chat.html} (81%) rename litellm/proxy/_experimental/out/experimental/{api-playground/index.html => api-playground.html} (97%) rename litellm/proxy/_experimental/out/experimental/{budgets/index.html => budgets.html} (96%) rename litellm/proxy/_experimental/out/experimental/{caching/index.html => caching.html} (95%) rename litellm/proxy/_experimental/out/experimental/{claude-code-plugins/index.html => claude-code-plugins.html} (96%) rename litellm/proxy/_experimental/out/experimental/{old-usage/index.html => old-usage.html} (95%) rename litellm/proxy/_experimental/out/experimental/{prompts/index.html => prompts.html} (96%) rename litellm/proxy/_experimental/out/experimental/{tag-management/index.html => tag-management.html} (97%) rename litellm/proxy/_experimental/out/{guardrails/index.html => guardrails.html} (97%) rename litellm/proxy/_experimental/out/{login/index.html => login.html} (96%) rename litellm/proxy/_experimental/out/{logs/index.html => logs.html} (84%) rename litellm/proxy/_experimental/out/mcp/oauth/{callback/index.html => callback.html} (96%) rename litellm/proxy/_experimental/out/{model-hub/index.html => model-hub.html} (96%) rename litellm/proxy/_experimental/out/{model_hub/index.html => model_hub.html} (95%) rename litellm/proxy/_experimental/out/{model_hub_table/index.html => model_hub_table.html} (94%) rename litellm/proxy/_experimental/out/{models-and-endpoints/index.html => models-and-endpoints.html} (92%) rename litellm/proxy/_experimental/out/{onboarding/index.html => onboarding.html} (96%) rename litellm/proxy/_experimental/out/{organizations/index.html => organizations.html} (97%) rename litellm/proxy/_experimental/out/{playground/index.html => playground.html} (93%) rename litellm/proxy/_experimental/out/{policies/index.html => policies.html} (96%) rename litellm/proxy/_experimental/out/settings/{admin-settings/index.html => admin-settings.html} (96%) rename litellm/proxy/_experimental/out/settings/{logging-and-alerts/index.html => logging-and-alerts.html} (94%) rename litellm/proxy/_experimental/out/settings/{router-settings/index.html => router-settings.html} (92%) rename litellm/proxy/_experimental/out/settings/{ui-theme/index.html => ui-theme.html} (97%) rename litellm/proxy/_experimental/out/{skills/index.html => skills.html} (97%) rename litellm/proxy/_experimental/out/{teams/index.html => teams.html} (93%) rename litellm/proxy/_experimental/out/{test-key/index.html => test-key.html} (95%) rename litellm/proxy/_experimental/out/tools/{mcp-servers/index.html => mcp-servers.html} (91%) rename litellm/proxy/_experimental/out/tools/{vector-stores/index.html => vector-stores.html} (96%) rename litellm/proxy/_experimental/out/{usage/index.html => usage.html} (92%) rename litellm/proxy/_experimental/out/{users/index.html => users.html} (97%) rename litellm/proxy/_experimental/out/{virtual-keys/index.html => virtual-keys.html} (97%) diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404.html similarity index 98% rename from litellm/proxy/_experimental/out/404/index.html rename to litellm/proxy/_experimental/out/404.html index 46e13ca9931b..38a2c3bd836e 100644 --- a/litellm/proxy/_experimental/out/404/index.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/__next.__PAGE__.txt b/litellm/proxy/_experimental/out/__next.__PAGE__.txt index 29fe35675022..18bda7f10651 100644 --- a/litellm/proxy/_experimental/out/__next.__PAGE__.txt +++ b/litellm/proxy/_experimental/out/__next.__PAGE__.txt @@ -1,30 +1,30 @@ 1:"$Sreact.fragment" 2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js"],"default"] +3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js"],"default"] 1a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] 1b:"$Sreact.suspense" :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false} +0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false} 4:{} 5:"$0:rsc:props:children:0:props:serverProvidedParams:params" 6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}] -7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}] +7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","async":true}] 8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","async":true}] 9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}] a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}] -b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","async":true}] +b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","async":true}] c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true}] -d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","async":true}] +d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}] e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}] -f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","async":true}] -10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","async":true}] +f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}] +10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","async":true}] 11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}] 12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}] -13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","async":true}] -14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","async":true}] -15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","async":true}] -16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","async":true}] -17:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}] -18:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js","async":true}] +13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","async":true}] +14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}] +15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","async":true}] +16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","async":true}] +17:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","async":true}] +18:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js","async":true}] 19:["$","$L1a",null,{"children":["$","$1b",null,{"name":"Next.MetadataOutlet","children":"$@1c"}]}] 1c:null diff --git a/litellm/proxy/_experimental/out/__next._full.txt b/litellm/proxy/_experimental/out/__next._full.txt index 33e1ef61e1fa..d213f7190c49 100644 --- a/litellm/proxy/_experimental/out/__next._full.txt +++ b/litellm/proxy/_experimental/out/__next._full.txt @@ -4,54 +4,54 @@ 4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 6:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -7:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js"],"default"] +7:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js"],"default"] 31:I[168027,[],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] :HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"P":null,"b":"LpD6ruZoEpvYpT5IvMEoa","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true,"nonce":"$undefined"}],"$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18","$L19","$L1a","$L1b","$L1c","$L1d","$L1e","$L1f","$L20","$L21","$L22","$L23","$L24","$L25","$L26","$L27","$L28","$L29","$L2a","$L2b","$L2c","$L2d","$L2e"],"$L2f"]}],{},null,false,false]},null,false,false],"$L30",false]],"m":"$undefined","G":["$31",[]],"S":true} +0:{"P":null,"b":"wnL6e5S6xaG1UdkxtYrTo","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true,"nonce":"$undefined"}],"$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18","$L19","$L1a","$L1b","$L1c","$L1d","$L1e","$L1f","$L20","$L21","$L22","$L23","$L24","$L25","$L26","$L27","$L28","$L29","$L2a","$L2b","$L2c","$L2d","$L2e"],"$L2f"]}],{},null,false,false]},null,false,false],"$L30",false]],"m":"$undefined","G":["$31",[]],"S":true} 32:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] 33:"$Sreact.suspense" 35:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] 37:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] -a:["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","async":true,"nonce":"$undefined"}] -b:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true,"nonce":"$undefined"}] +a:["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/0974abc09c5e7ada.js","async":true,"nonce":"$undefined"}] +b:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ae625aa52246581e.js","async":true,"nonce":"$undefined"}] c:["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true,"nonce":"$undefined"}] d:["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true,"nonce":"$undefined"}] e:["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true,"nonce":"$undefined"}] f:["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true,"nonce":"$undefined"}] 10:["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true,"nonce":"$undefined"}] -11:["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true,"nonce":"$undefined"}] -12:["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true,"nonce":"$undefined"}] +11:["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/88001a7ecaf7b1af.js","async":true,"nonce":"$undefined"}] +12:["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/cbc99c8fae110c02.js","async":true,"nonce":"$undefined"}] 13:["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true,"nonce":"$undefined"}] 14:["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true,"nonce":"$undefined"}] -15:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true,"nonce":"$undefined"}] -16:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","async":true,"nonce":"$undefined"}] -17:["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","async":true,"nonce":"$undefined"}] +15:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/1c881baaaa68b7a5.js","async":true,"nonce":"$undefined"}] +16:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9955c118354ef6cc.js","async":true,"nonce":"$undefined"}] +17:["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true,"nonce":"$undefined"}] 18:["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true,"nonce":"$undefined"}] -19:["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}] +19:["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true,"nonce":"$undefined"}] 1a:["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true,"nonce":"$undefined"}] -1b:["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true,"nonce":"$undefined"}] +1b:["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/5181a28310842d3d.js","async":true,"nonce":"$undefined"}] 1c:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true,"nonce":"$undefined"}] -1d:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true,"nonce":"$undefined"}] +1d:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/16a1651c0b3e7c8e.js","async":true,"nonce":"$undefined"}] 1e:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","async":true,"nonce":"$undefined"}] 1f:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true,"nonce":"$undefined"}] 20:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true,"nonce":"$undefined"}] -21:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","async":true,"nonce":"$undefined"}] +21:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/631b1874cba557c9.js","async":true,"nonce":"$undefined"}] 22:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true,"nonce":"$undefined"}] -23:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","async":true,"nonce":"$undefined"}] +23:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true,"nonce":"$undefined"}] 24:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true,"nonce":"$undefined"}] -25:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","async":true,"nonce":"$undefined"}] -26:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","async":true,"nonce":"$undefined"}] +25:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}] +26:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/003f1ffc5817ab83.js","async":true,"nonce":"$undefined"}] 27:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true,"nonce":"$undefined"}] 28:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true,"nonce":"$undefined"}] -29:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","async":true,"nonce":"$undefined"}] -2a:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","async":true,"nonce":"$undefined"}] -2b:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","async":true,"nonce":"$undefined"}] -2c:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","async":true,"nonce":"$undefined"}] -2d:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true,"nonce":"$undefined"}] -2e:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js","async":true,"nonce":"$undefined"}] +29:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/23bfdf9b0544f0b1.js","async":true,"nonce":"$undefined"}] +2a:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true,"nonce":"$undefined"}] +2b:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/726bebeef472c6cb.js","async":true,"nonce":"$undefined"}] +2c:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","async":true,"nonce":"$undefined"}] +2d:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","async":true,"nonce":"$undefined"}] +2e:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/16c0e58809eaf2b5.js","async":true,"nonce":"$undefined"}] 2f:["$","$L32",null,{"children":["$","$33",null,{"name":"Next.MetadataOutlet","children":"$@34"}]}] 30:["$","$1","h",{"children":[null,["$","$L35",null,{"children":"$L36"}],["$","div",null,{"hidden":true,"children":["$","$L37",null,{"children":["$","$33",null,{"name":"Next.Metadata","children":"$L38"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] 8:{} diff --git a/litellm/proxy/_experimental/out/__next._head.txt b/litellm/proxy/_experimental/out/__next._head.txt index 52cb2daa0385..82758aa5c3f6 100644 --- a/litellm/proxy/_experimental/out/__next._head.txt +++ b/litellm/proxy/_experimental/out/__next._head.txt @@ -3,4 +3,4 @@ 3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] 4:"$Sreact.suspense" 5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] -0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} +0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._index.txt b/litellm/proxy/_experimental/out/__next._index.txt index 24ed6776f930..545ff2e55ccf 100644 --- a/litellm/proxy/_experimental/out/__next._index.txt +++ b/litellm/proxy/_experimental/out/__next._index.txt @@ -5,4 +5,4 @@ 5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] :HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"] -0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false} +0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._tree.txt b/litellm/proxy/_experimental/out/__next._tree.txt index 78aafc1b3f50..10f5e5c27213 100644 --- a/litellm/proxy/_experimental/out/__next._tree.txt +++ b/litellm/proxy/_experimental/out/__next._tree.txt @@ -2,4 +2,4 @@ :HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} +0:{"buildId":"wnL6e5S6xaG1UdkxtYrTo","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/3c0e9dc19dbbd4ed.js b/litellm/proxy/_experimental/out/_next/static/chunks/003f1ffc5817ab83.js similarity index 55% rename from litellm/proxy/_experimental/out/_next/static/chunks/3c0e9dc19dbbd4ed.js rename to litellm/proxy/_experimental/out/_next/static/chunks/003f1ffc5817ab83.js index e94c4ad8fcae..0311d4a524cf 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/3c0e9dc19dbbd4ed.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/003f1ffc5817ab83.js @@ -1 +1 @@ -(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,91979,e=>{"use strict";e.i(247167);var t=e.i(931067),l=e.i(271645);let a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M909.1 209.3l-56.4 44.1C775.8 155.1 656.2 92 521.9 92 290 92 102.3 279.5 102 511.5 101.7 743.7 289.8 932 521.9 932c181.3 0 335.8-115 394.6-276.1 1.5-4.2-.7-8.9-4.9-10.3l-56.7-19.5a8 8 0 00-10.1 4.8c-1.8 5-3.8 10-5.9 14.9-17.3 41-42.1 77.8-73.7 109.4A344.77 344.77 0 01655.9 829c-42.3 17.9-87.4 27-133.8 27-46.5 0-91.5-9.1-133.8-27A341.5 341.5 0 01279 755.2a342.16 342.16 0 01-73.7-109.4c-17.9-42.4-27-87.4-27-133.9s9.1-91.5 27-133.9c17.3-41 42.1-77.8 73.7-109.4 31.6-31.6 68.4-56.4 109.3-73.8 42.3-17.9 87.4-27 133.8-27 46.5 0 91.5 9.1 133.8 27a341.5 341.5 0 01109.3 73.8c9.9 9.9 19.2 20.4 27.8 31.4l-60.2 47a8 8 0 003 14.1l175.6 43c5 1.2 9.9-2.6 9.9-7.7l.8-180.9c-.1-6.6-7.8-10.3-13-6.2z"}}]},name:"reload",theme:"outlined"};var r=e.i(9583),s=l.forwardRef(function(e,s){return l.createElement(r.default,(0,t.default)({},e,{ref:s,icon:a}))});e.s(["ReloadOutlined",0,s],91979)},625901,e=>{"use strict";var t=e.i(266027),l=e.i(621482),a=e.i(243652),r=e.i(764205),s=e.i(135214);let i=(0,a.createQueryKeys)("models"),n=(0,a.createQueryKeys)("modelHub"),o=(0,a.createQueryKeys)("allProxyModels");(0,a.createQueryKeys)("selectedTeamModels");let d=(0,a.createQueryKeys)("infiniteModels");e.s(["useAllProxyModels",0,()=>{let{accessToken:e,userId:l,userRole:a}=(0,s.default)();return(0,t.useQuery)({queryKey:o.list({}),queryFn:async()=>await (0,r.modelAvailableCall)(e,l,a,!0,null,!0,!1,"expand"),enabled:!!(e&&l&&a)})},"useInfiniteModelInfo",0,(e=50,t)=>{let{accessToken:a,userId:i,userRole:n}=(0,s.default)();return(0,l.useInfiniteQuery)({queryKey:d.list({filters:{...i&&{userId:i},...n&&{userRole:n},size:e,...t&&{search:t}}}),queryFn:async({pageParam:l})=>await (0,r.modelInfoCall)(a,i,n,l,e,t),initialPageParam:1,getNextPageParam:e=>{if(e.current_page{let{accessToken:e}=(0,s.default)();return(0,t.useQuery)({queryKey:n.list({}),queryFn:async()=>await (0,r.modelHubCall)(e),enabled:!!e})},"useModelsInfo",0,(e=1,l=50,a,n,o,d,m)=>{let{accessToken:c,userId:u,userRole:g}=(0,s.default)();return(0,t.useQuery)({queryKey:i.list({filters:{...u&&{userId:u},...g&&{userRole:g},page:e,size:l,...a&&{search:a},...n&&{modelId:n},...o&&{teamId:o},...d&&{sortBy:d},...m&&{sortOrder:m}}}),queryFn:async()=>await (0,r.modelInfoCall)(c,u,g,e,l,a,n,o,d,m),enabled:!!(c&&u&&g)})}])},969550,e=>{"use strict";var t=e.i(843476),l=e.i(271645);let a=l.forwardRef(function(e,t){return l.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:t},e),l.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M3 4a1 1 0 011-1h16a1 1 0 011 1v2.586a1 1 0 01-.293.707l-6.414 6.414a1 1 0 00-.293.707V17l-4 4v-6.586a1 1 0 00-.293-.707L3.293 7.293A1 1 0 013 6.586V4z"}))});var r=e.i(464571),s=e.i(311451),i=e.i(199133),n=e.i(374009);e.s(["default",0,({options:e,onApplyFilters:o,onResetFilters:d,initialValues:m={},buttonLabel:c="Filters"})=>{let[u,g]=(0,l.useState)(!1),[h,p]=(0,l.useState)(m),[x,b]=(0,l.useState)({}),[_,f]=(0,l.useState)({}),[y,j]=(0,l.useState)({}),[v,w]=(0,l.useState)({}),C=(0,l.useCallback)((0,n.default)(async(e,t)=>{if(t.isSearchable&&t.searchFn){f(e=>({...e,[t.name]:!0}));try{let l=await t.searchFn(e);b(e=>({...e,[t.name]:l}))}catch(e){console.error("Error searching:",e),b(e=>({...e,[t.name]:[]}))}finally{f(e=>({...e,[t.name]:!1}))}}},300),[]),S=(0,l.useCallback)(async e=>{if(e.isSearchable&&e.searchFn&&!v[e.name]){f(t=>({...t,[e.name]:!0})),w(t=>({...t,[e.name]:!0}));try{let t=await e.searchFn("");b(l=>({...l,[e.name]:t}))}catch(t){console.error("Error loading initial options:",t),b(t=>({...t,[e.name]:[]}))}finally{f(t=>({...t,[e.name]:!1}))}}},[v]);(0,l.useEffect)(()=>{u&&e.forEach(e=>{e.isSearchable&&!v[e.name]&&S(e)})},[u,e,S,v]);let T=(e,t)=>{let l={...h,[e]:t};p(l),o(l)};return(0,t.jsxs)("div",{className:"w-full",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 mb-6",children:[(0,t.jsx)(r.Button,{icon:(0,t.jsx)(a,{className:"h-4 w-4"}),onClick:()=>g(!u),className:"flex items-center gap-2",children:c}),(0,t.jsx)(r.Button,{onClick:()=>{let t={};e.forEach(e=>{t[e.name]=""}),p(t),d()},children:"Reset Filters"})]}),u&&(0,t.jsx)("div",{className:"grid grid-cols-3 gap-x-6 gap-y-4 mb-6",children:["Team ID","Status","Organization ID","Key Alias","User ID","End User","Error Code","Error Message","Key Hash","Model","Public model / search tool"].map(l=>{let a,r=e.find(e=>e.label===l||e.name===l);return r?(0,t.jsxs)("div",{className:"flex flex-col gap-2",children:[(0,t.jsx)("label",{className:"text-sm text-gray-600",children:r.label||r.name}),r.isSearchable?(0,t.jsx)(i.Select,{showSearch:!0,className:"w-full",placeholder:`Search ${r.label||r.name}...`,value:h[r.name]||void 0,onChange:e=>T(r.name,e),onOpenChange:e=>{e&&r.isSearchable&&!v[r.name]&&S(r)},onSearch:e=>{j(t=>({...t,[r.name]:e})),r.searchFn&&C(e,r)},filterOption:!1,loading:_[r.name],options:x[r.name]||[],allowClear:!0,notFoundContent:_[r.name]?"Loading...":"No results found"}):r.options?(0,t.jsx)(i.Select,{className:"w-full",placeholder:`Select ${r.label||r.name}...`,value:h[r.name]||void 0,onChange:e=>T(r.name,e),allowClear:!0,children:r.options.map(e=>(0,t.jsx)(i.Select.Option,{value:e.value,children:e.label},e.value))}):r.customComponent?(a=r.customComponent,(0,t.jsx)(a,{value:h[r.name]||void 0,onChange:e=>T(r.name,e??""),placeholder:`Select ${r.label||r.name}...`,allFilters:h})):(0,t.jsx)(s.Input,{className:"w-full",placeholder:`Enter ${r.label||r.name}...`,value:h[r.name]||"",onChange:e=>T(r.name,e.target.value),allowClear:!0})]},r.name):null})})]})}],969550)},633627,e=>{"use strict";var t=e.i(764205);let l=(e,t,l,a)=>{for(let r of e){let e=r?.key_alias;e&&"string"==typeof e&&t.add(e.trim());let s=r?.organization_id??r?.org_id;s&&"string"==typeof s&&l.add(s.trim());let i=r?.user_id;if(i&&"string"==typeof i){let e=r?.user?.user_email||i;a.set(i,e)}}},a=async(e,a)=>{if(!e||!a)return{keyAliases:[],organizationIds:[],userIds:[]};try{let r=new Set,s=new Set,i=new Map,n=await (0,t.keyListCall)(e,null,a,null,null,null,1,100,null,null,"user",null),o=n?.keys||[],d=n?.total_pages??1;l(o,r,s,i);let m=Math.min(d,10)-1;if(m>0){let n=Array.from({length:m},(l,r)=>(0,t.keyListCall)(e,null,a,null,null,null,r+2,100,null,null,"user",null));for(let e of(await Promise.allSettled(n)))"fulfilled"===e.status&&l(e.value?.keys||[],r,s,i)}return{keyAliases:Array.from(r).sort(),organizationIds:Array.from(s).sort(),userIds:Array.from(i.entries()).map(([e,t])=>({id:e,email:t}))}}catch(e){return console.error("Error fetching team filter options:",e),{keyAliases:[],organizationIds:[],userIds:[]}}},r=async(e,l)=>{if(!e)return[];try{let a=[],r=1,s=!0;for(;s;){let i=await (0,t.teamListCall)(e,l||null,null);a=[...a,...i],r{if(!e)return[];try{let l=[],a=1,r=!0;for(;r;){let s=await (0,t.organizationListCall)(e);l=[...l,...s],a{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3"}))});e.s(["ClipboardCopyIcon",0,l],551332)},122577,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M14.752 11.168l-3.197-2.132A1 1 0 0010 9.87v4.263a1 1 0 001.555.832l3.197-2.132a1 1 0 000-1.664z"}),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M21 12a9 9 0 11-18 0 9 9 0 0118 0z"}))});e.s(["PlayIcon",0,l],122577)},902555,e=>{"use strict";var t=e.i(843476),l=e.i(591935),a=e.i(122577),r=e.i(278587),s=e.i(68155),i=e.i(360820),n=e.i(871943),o=e.i(434626),d=e.i(551332),m=e.i(592968),c=e.i(115504),u=e.i(752978);function g({icon:e,onClick:l,className:a,disabled:r,dataTestId:s}){return r?(0,t.jsx)(u.Icon,{icon:e,size:"sm",className:"opacity-50 cursor-not-allowed","data-testid":s}):(0,t.jsx)(u.Icon,{icon:e,size:"sm",onClick:l,className:(0,c.cx)("cursor-pointer",a),"data-testid":s})}let h={Edit:{icon:l.PencilAltIcon,className:"hover:text-blue-600"},Delete:{icon:s.TrashIcon,className:"hover:text-red-600"},Test:{icon:a.PlayIcon,className:"hover:text-blue-600"},Regenerate:{icon:r.RefreshIcon,className:"hover:text-green-600"},Up:{icon:i.ChevronUpIcon,className:"hover:text-blue-600"},Down:{icon:n.ChevronDownIcon,className:"hover:text-blue-600"},Open:{icon:o.ExternalLinkIcon,className:"hover:text-green-600"},Copy:{icon:d.ClipboardCopyIcon,className:"hover:text-blue-600"}};function p({onClick:e,tooltipText:l,disabled:a=!1,disabledTooltipText:r,dataTestId:s,variant:i}){let{icon:n,className:o}=h[i];return(0,t.jsx)(m.Tooltip,{title:a?r:l,children:(0,t.jsx)("span",{children:(0,t.jsx)(g,{icon:n,onClick:e,className:o,disabled:a,dataTestId:s})})})}e.s(["default",()=>p],902555)},434626,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14"}))});e.s(["ExternalLinkIcon",0,l],434626)},278587,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"}))});e.s(["RefreshIcon",0,l],278587)},207670,e=>{"use strict";function t(){for(var e,t,l=0,a="",r=arguments.length;lt,"default",0,t])},728889,e=>{"use strict";var t=e.i(290571),l=e.i(271645),a=e.i(829087),r=e.i(480731),s=e.i(444755),i=e.i(673706),n=e.i(95779);let o={xs:{paddingX:"px-1.5",paddingY:"py-1.5"},sm:{paddingX:"px-1.5",paddingY:"py-1.5"},md:{paddingX:"px-2",paddingY:"py-2"},lg:{paddingX:"px-2",paddingY:"py-2"},xl:{paddingX:"px-2.5",paddingY:"py-2.5"}},d={xs:{height:"h-3",width:"w-3"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-7",width:"w-7"},xl:{height:"h-9",width:"w-9"}},m={simple:{rounded:"",border:"",ring:"",shadow:""},light:{rounded:"rounded-tremor-default",border:"",ring:"",shadow:""},shadow:{rounded:"rounded-tremor-default",border:"border",ring:"",shadow:"shadow-tremor-card dark:shadow-dark-tremor-card"},solid:{rounded:"rounded-tremor-default",border:"border-2",ring:"ring-1",shadow:""},outlined:{rounded:"rounded-tremor-default",border:"border",ring:"ring-2",shadow:""}},c=(0,i.makeClassName)("Icon"),u=l.default.forwardRef((e,u)=>{let{icon:g,variant:h="simple",tooltip:p,size:x=r.Sizes.SM,color:b,className:_}=e,f=(0,t.__rest)(e,["icon","variant","tooltip","size","color","className"]),y=((e,t)=>{switch(e){case"simple":return{textColor:t?(0,i.getColorClassNames)(t,n.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:"",borderColor:"",ringColor:""};case"light":return{textColor:t?(0,i.getColorClassNames)(t,n.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,n.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand-muted dark:bg-dark-tremor-brand-muted",borderColor:"",ringColor:""};case"shadow":return{textColor:t?(0,i.getColorClassNames)(t,n.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,n.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:"border-tremor-border dark:border-dark-tremor-border",ringColor:""};case"solid":return{textColor:t?(0,i.getColorClassNames)(t,n.colorPalette.text).textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,n.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand dark:bg-dark-tremor-brand",borderColor:"border-tremor-brand-inverted dark:border-dark-tremor-brand-inverted",ringColor:"ring-tremor-ring dark:ring-dark-tremor-ring"};case"outlined":return{textColor:t?(0,i.getColorClassNames)(t,n.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,n.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:t?(0,i.getColorClassNames)(t,n.colorPalette.ring).borderColor:"border-tremor-brand-subtle dark:border-dark-tremor-brand-subtle",ringColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,n.colorPalette.ring).ringColor,"ring-opacity-40"):"ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted"}}})(h,b),{tooltipProps:j,getReferenceProps:v}=(0,a.useTooltip)();return l.default.createElement("span",Object.assign({ref:(0,i.mergeRefs)([u,j.refs.setReference]),className:(0,s.tremorTwMerge)(c("root"),"inline-flex shrink-0 items-center justify-center",y.bgColor,y.textColor,y.borderColor,y.ringColor,m[h].rounded,m[h].border,m[h].shadow,m[h].ring,o[x].paddingX,o[x].paddingY,_)},v,f),l.default.createElement(a.default,Object.assign({text:p},j)),l.default.createElement(g,{className:(0,s.tremorTwMerge)(c("icon"),"shrink-0",d[x].height,d[x].width)}))});u.displayName="Icon",e.s(["default",()=>u],728889)},752978,e=>{"use strict";var t=e.i(728889);e.s(["Icon",()=>t.default])},591935,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"}))});e.s(["PencilAltIcon",0,l],591935)},907308,e=>{"use strict";var t=e.i(843476),l=e.i(271645),a=e.i(212931),r=e.i(808613),s=e.i(464571),i=e.i(199133),n=e.i(592968),o=e.i(213205),d=e.i(374009),m=e.i(764205);e.s(["default",0,({isVisible:e,onCancel:c,onSubmit:u,accessToken:g,title:h="Add Team Member",roles:p=[{label:"admin",value:"admin",description:"Admin role. Can create team keys, add members, and manage settings."},{label:"user",value:"user",description:"User role. Can view team info, but not manage it."}],defaultRole:x="user",teamId:b})=>{let[_]=r.Form.useForm(),[f,y]=(0,l.useState)([]),[j,v]=(0,l.useState)(!1),[w,C]=(0,l.useState)("user_email"),[S,T]=(0,l.useState)(!1),N=async(e,t)=>{if(!e)return void y([]);v(!0);try{let l=new URLSearchParams;if(l.append(t,e),b&&l.append("team_id",b),null==g)return;let a=(await (0,m.userFilterUICall)(g,l)).map(e=>({label:"user_email"===t?`${e.user_email}`:`${e.user_id}`,value:"user_email"===t?e.user_email:e.user_id,user:e}));y(a)}catch(e){console.error("Error fetching users:",e)}finally{v(!1)}},k=(0,l.useCallback)((0,d.default)((e,t)=>N(e,t),300),[]),I=(e,t)=>{C(t),k(e,t)},M=(e,t)=>{let l=t.user;_.setFieldsValue({user_email:l.user_email,user_id:l.user_id,role:_.getFieldValue("role")})},A=async e=>{T(!0);try{await u(e)}finally{T(!1)}};return(0,t.jsx)(a.Modal,{title:h,open:e,onCancel:()=>{_.resetFields(),y([]),c()},footer:null,width:800,maskClosable:!S,children:(0,t.jsxs)(r.Form,{form:_,onFinish:A,labelCol:{span:8},wrapperCol:{span:16},labelAlign:"left",initialValues:{role:x},children:[(0,t.jsx)(r.Form.Item,{label:"Email",name:"user_email",className:"mb-4",children:(0,t.jsx)(i.Select,{showSearch:!0,className:"w-full",placeholder:"Search by email",filterOption:!1,onSearch:e=>I(e,"user_email"),onSelect:(e,t)=>M(e,t),options:"user_email"===w?f:[],loading:j,allowClear:!0,"data-testid":"member-email-search"})}),(0,t.jsx)("div",{className:"text-center mb-4",children:"OR"}),(0,t.jsx)(r.Form.Item,{label:"User ID",name:"user_id",className:"mb-4",children:(0,t.jsx)(i.Select,{showSearch:!0,className:"w-full",placeholder:"Search by user ID",filterOption:!1,onSearch:e=>I(e,"user_id"),onSelect:(e,t)=>M(e,t),options:"user_id"===w?f:[],loading:j,allowClear:!0})}),(0,t.jsx)(r.Form.Item,{label:"Member Role",name:"role",className:"mb-4",children:(0,t.jsx)(i.Select,{defaultValue:x,children:p.map(e=>(0,t.jsx)(i.Select.Option,{value:e.value,children:(0,t.jsxs)(n.Tooltip,{title:e.description,children:[(0,t.jsx)("span",{className:"font-medium",children:e.label}),(0,t.jsxs)("span",{className:"ml-2 text-gray-500 text-sm",children:["- ",e.description]})]})},e.value))})}),(0,t.jsx)("div",{className:"text-right mt-4",children:(0,t.jsx)(s.Button,{type:"primary",htmlType:"submit",icon:(0,t.jsx)(o.UserAddOutlined,{}),loading:S,children:S?"Adding...":"Add Member"})})]})})}])},162386,e=>{"use strict";var t=e.i(843476),l=e.i(625901),a=e.i(109799),r=e.i(785242),s=e.i(738014),i=e.i(199133),n=e.i(981339),o=e.i(592968);let d={label:"All Proxy Models",value:"all-proxy-models"},m={label:"No Default Models",value:"no-default-models"},c=[d,m],u={user:({allProxyModels:e,userModels:t,options:l})=>t&&l?.includeUserModels?t:[],team:({allProxyModels:e,selectedOrganization:t,userModels:l})=>t?t.models.includes(d.value)||0===t.models.length?e:e.filter(e=>t.models.includes(e)):e??[],organization:({allProxyModels:e})=>e,global:({allProxyModels:e})=>e};e.s(["ModelSelect",0,e=>{let{teamID:g,organizationID:h,options:p,context:x,dataTestId:b,value:_=[],onChange:f,style:y}=e,{includeUserModels:j,showAllTeamModelsOption:v,showAllProxyModelsOverride:w,includeSpecialOptions:C}=p||{},{data:S,isLoading:T}=(0,l.useAllProxyModels)(),{data:N,isLoading:k}=(0,r.useTeam)(g),{data:I,isLoading:M}=(0,a.useOrganization)(h),{data:A,isLoading:F}=(0,s.useCurrentUser)(),O=e=>c.some(t=>t.value===e),z=_.some(O),P=I?.models.includes(d.value)||I?.models.length===0;if(T||k||M||F)return(0,t.jsx)(n.Skeleton.Input,{active:!0,block:!0});let{wildcard:L,regular:D}=(e=>{let t=[],l=[];for(let a of e)a.endsWith("/*")?t.push(a):l.push(a);return{wildcard:t,regular:l}})(((e,t,l)=>{let a=Array.from(new Map(e.map(e=>[e.id,e])).values()).map(e=>e.id);if(t.options?.showAllProxyModelsOverride)return a;let r=u[t.context];return r?r({allProxyModels:a,...l,options:t.options}):[]})(S?.data??[],e,{selectedTeam:N,selectedOrganization:I,userModels:A?.models}));return(0,t.jsx)(i.Select,{"data-testid":b,value:_,onChange:e=>{let t=e.filter(O);f(t.length>0?[t[t.length-1]]:e)},style:y,options:[...C?[{label:(0,t.jsx)("span",{children:"Special Options"}),title:"Special Options",options:[...w||P&&C||"global"===x?[{label:(0,t.jsx)("span",{children:"All Proxy Models"}),value:d.value,disabled:_.length>0&&_.some(e=>O(e)&&e!==d.value),key:d.value}]:[],{label:(0,t.jsx)("span",{children:"No Default Models"}),value:m.value,disabled:_.length>0&&_.some(e=>O(e)&&e!==m.value),key:m.value}]}]:[],...L.length>0?[{label:(0,t.jsx)("span",{children:"Wildcard Options"}),title:"Wildcard Options",options:L.map(e=>{let l=e.replace("/*",""),a=l.charAt(0).toUpperCase()+l.slice(1);return{label:(0,t.jsx)("span",{children:`All ${a} models`}),value:e,disabled:z}})}]:[],{label:(0,t.jsx)("span",{children:"Models"}),title:"Models",options:D.map(e=>({label:(0,t.jsx)("span",{children:e}),value:e,disabled:z}))}],mode:"multiple",placeholder:"Select Models",allowClear:!0,maxTagCount:"responsive",maxTagPlaceholder:e=>(0,t.jsx)(o.Tooltip,{styles:{root:{pointerEvents:"none"}},title:e.map(({value:e})=>e).join(", "),children:(0,t.jsxs)("span",{children:["+",e.length," more"]})})})}],162386)},276173,e=>{"use strict";var t=e.i(843476),l=e.i(599724),a=e.i(779241),r=e.i(464571),s=e.i(808613),i=e.i(212931),n=e.i(199133),o=e.i(271645),d=e.i(435451);e.s(["default",0,({visible:e,onCancel:m,onSubmit:c,initialData:u,mode:g,config:h})=>{let p,[x]=s.Form.useForm(),[b,_]=(0,o.useState)(!1);console.log("Initial Data:",u),(0,o.useEffect)(()=>{if(e)if("edit"===g&&u){let e={...u,role:u.role||h.defaultRole,max_budget_in_team:u.max_budget_in_team||null,tpm_limit:u.tpm_limit||null,rpm_limit:u.rpm_limit||null,allowed_models:u.allowed_models||[]};console.log("Setting form values:",e),x.setFieldsValue(e)}else x.resetFields(),x.setFieldsValue({role:h.defaultRole||h.roleOptions[0]?.value})},[e,u,g,x,h.defaultRole,h.roleOptions]);let f=async e=>{try{_(!0);let t=Object.entries(e).reduce((e,[t,l])=>{if("string"==typeof l){let a=l.trim();return""===a&&("max_budget_in_team"===t||"tpm_limit"===t||"rpm_limit"===t)?{...e,[t]:null}:{...e,[t]:a}}return{...e,[t]:l}},{});console.log("Submitting form data:",t),await Promise.resolve(c(t)),x.resetFields()}catch(e){console.error("Form submission error:",e)}finally{_(!1)}};return(0,t.jsx)(i.Modal,{title:h.title||("add"===g?"Add Member":"Edit Member"),open:e,width:1e3,footer:null,onCancel:m,children:(0,t.jsxs)(s.Form,{form:x,onFinish:f,labelCol:{span:8},wrapperCol:{span:16},labelAlign:"left",children:[h.showEmail&&(0,t.jsx)(s.Form.Item,{label:"Email",name:"user_email",className:"mb-4",rules:[{type:"email",message:"Please enter a valid email!"}],children:(0,t.jsx)(a.TextInput,{placeholder:"user@example.com"})}),h.showEmail&&h.showUserId&&(0,t.jsx)("div",{className:"text-center mb-4",children:(0,t.jsx)(l.Text,{children:"OR"})}),h.showUserId&&(0,t.jsx)(s.Form.Item,{label:"User ID",name:"user_id",className:"mb-4",children:(0,t.jsx)(a.TextInput,{placeholder:"user_123"})}),(0,t.jsx)(s.Form.Item,{label:(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)("span",{children:"Role"}),"edit"===g&&u&&(0,t.jsxs)("span",{className:"text-gray-500 text-sm",children:["(Current: ",(p=u.role,h.roleOptions.find(e=>e.value===p)?.label||p),")"]})]}),name:"role",className:"mb-4",rules:[{required:!0,message:"Please select a role!"}],children:(0,t.jsx)(n.Select,{children:"edit"===g&&u?[...h.roleOptions.filter(e=>e.value===u.role),...h.roleOptions.filter(e=>e.value!==u.role)].map(e=>(0,t.jsx)(n.Select.Option,{value:e.value,children:e.label},e.value)):h.roleOptions.map(e=>(0,t.jsx)(n.Select.Option,{value:e.value,children:e.label},e.value))})}),h.additionalFields?.map(e=>(0,t.jsx)(s.Form.Item,{label:e.label,name:e.name,className:"mb-4",rules:e.rules,children:(e=>{switch(e.type){case"input":return(0,t.jsx)(a.TextInput,{placeholder:e.placeholder});case"numerical":return(0,t.jsx)(d.default,{step:e.step||1,min:e.min||0,style:{width:"100%"},placeholder:e.placeholder||"Enter a numerical value"});case"select":return(0,t.jsx)(n.Select,{children:e.options?.map(e=>(0,t.jsx)(n.Select.Option,{value:e.value,children:e.label},e.value))});case"multi-select":return(0,t.jsx)(n.Select,{mode:"multiple",placeholder:e.placeholder||"Select options",options:e.options,allowClear:!0});default:return null}})(e)},e.name)),(0,t.jsxs)("div",{className:"text-right mt-6",children:[(0,t.jsx)(r.Button,{onClick:m,className:"mr-2",disabled:b,children:"Cancel"}),(0,t.jsx)(r.Button,{type:"default",htmlType:"submit",loading:b,children:"add"===g?b?"Adding...":"Add Member":b?"Saving...":"Save Changes"})]})]})})}])},294612,e=>{"use strict";var t=e.i(843476),l=e.i(100486),a=e.i(827252),r=e.i(213205),s=e.i(771674),i=e.i(464571),n=e.i(770914),o=e.i(291542),d=e.i(262218),m=e.i(592968),c=e.i(898586),u=e.i(902555);let{Text:g}=c.Typography;function h({members:e,canEdit:c,onEdit:h,onDelete:p,onAddMember:x,roleColumnTitle:b="Role",roleTooltip:_,extraColumns:f=[],showDeleteForMember:y,emptyText:j}){let v=[{title:"User Email",dataIndex:"user_email",key:"user_email",render:e=>(0,t.jsx)(g,{children:e||"-"})},{title:"User ID",dataIndex:"user_id",key:"user_id",render:e=>"default_user_id"===e?(0,t.jsx)(d.Tag,{color:"blue",children:"Default Proxy Admin"}):(0,t.jsx)(g,{children:e||"-"})},{title:_?(0,t.jsxs)(n.Space,{direction:"horizontal",children:[b,(0,t.jsx)(m.Tooltip,{title:_,children:(0,t.jsx)(a.InfoCircleOutlined,{})})]}):b,dataIndex:"role",key:"role",render:e=>(0,t.jsxs)(n.Space,{children:[e?.toLowerCase()==="admin"||e?.toLowerCase()==="org_admin"?(0,t.jsx)(l.CrownOutlined,{}):(0,t.jsx)(s.UserOutlined,{}),(0,t.jsx)(g,{style:{textTransform:"capitalize"},children:e||"-"})]})},...f,{title:"Actions",key:"actions",fixed:"right",width:120,render:(e,l)=>c?(0,t.jsxs)(n.Space,{children:[(0,t.jsx)(u.default,{variant:"Edit",tooltipText:"Edit member",dataTestId:"edit-member",onClick:()=>h(l)}),(!y||y(l))&&(0,t.jsx)(u.default,{variant:"Delete",tooltipText:"Delete member",dataTestId:"delete-member",onClick:()=>p(l)})]}):null}];return(0,t.jsxs)(n.Space,{direction:"vertical",style:{width:"100%"},children:[(0,t.jsxs)("span",{className:"inline-flex text-sm text-gray-700",children:[e.length," Member",1!==e.length?"s":""]}),(0,t.jsx)(o.Table,{columns:v,dataSource:e,rowKey:e=>e.user_id??e.user_email??JSON.stringify(e),pagination:!1,size:"small",scroll:{x:"max-content"},locale:j?{emptyText:j}:void 0}),x&&c&&(0,t.jsx)(i.Button,{icon:(0,t.jsx)(r.UserAddOutlined,{}),type:"primary",onClick:x,children:"Add Member"})]})}e.s(["default",()=>h])},56567,838932,471145,e=>{"use strict";var t=e.i(843476),l=e.i(135214),a=e.i(109799),r=e.i(912598),s=e.i(907308),i=e.i(764205),n=e.i(266027);let o=(0,e.i(243652).createQueryKeys)("guardrails"),d=()=>{let{accessToken:e,userId:t,userRole:a}=(0,l.default)();return(0,n.useQuery)({queryKey:o.list({}),queryFn:async()=>(0,i.getGuardrailsList)(e),enabled:!!(e&&t&&a),select:e=>{let t=e?.guardrails??[],l=new Set,a=new Set;for(let e of t)e.litellm_params?.default_on?l.add(e.guardrail_name):a.add(e.guardrail_name);return{guardrails:t,globalGuardrailNames:l,optionalGuardrailNames:a}}})};e.s(["useGuardrails",0,d],838932);var m=e.i(500330),c=e.i(11751),u=e.i(708347),g=e.i(751904),h=e.i(160818),p=e.i(827252),x=e.i(564897),b=e.i(646563),_=e.i(987432),f=e.i(530212),y=e.i(677667),j=e.i(130643),v=e.i(898667),w=e.i(389083),C=e.i(304967),S=e.i(350967),T=e.i(599724),N=e.i(779241),k=e.i(629569),I=e.i(464571),M=e.i(808613),A=e.i(311451),F=e.i(28651),O=e.i(199133),z=e.i(770914),P=e.i(790848),L=e.i(653496),D=e.i(262218),R=e.i(592968),E=e.i(888259),B=e.i(678784),U=e.i(118366),V=e.i(271645),K=e.i(9314),$=e.i(552130),G=e.i(127952);function W({className:e,value:l,onChange:a}){return(0,t.jsxs)(O.Select,{className:e,value:l,onChange:a,children:[(0,t.jsx)(O.Select.Option,{value:"24h",children:"Daily"}),(0,t.jsx)(O.Select.Option,{value:"7d",children:"Weekly"}),(0,t.jsx)(O.Select.Option,{value:"30d",children:"Monthly"})]})}var q=e.i(844565),H=e.i(355619);let Q=function({globalGuardrailNames:e,teamGuardrails:l=[],optedOutGlobalGuardrails:a=[],killSwitchOn:r=!1,variant:s="card",className:i=""}){let n=new Set(a),o=Array.from(e).filter(e=>!n.has(e)),d=l.filter(t=>!e.has(t)),m=r||0!==o.length||0!==d.length?(0,t.jsxs)("div",{className:"flex flex-col gap-4",children:[(0,t.jsxs)("div",{children:[(0,t.jsxs)("span",{className:"block text-sm font-medium text-gray-700 mb-2",children:[(0,t.jsx)(h.GlobalOutlined,{style:{marginInlineEnd:4},"aria-label":"Global guardrail"}),"Global"]}),r?(0,t.jsx)(D.Tag,{color:"gold",children:"Bypassed for this team"}):o.length>0?(0,t.jsx)("div",{className:"flex flex-wrap gap-2",children:o.map(e=>(0,t.jsx)(D.Tag,{color:"blue",children:e},e))}):(0,t.jsx)("span",{className:"block text-sm text-gray-500",children:"None configured"})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)("span",{className:"block text-sm font-medium text-gray-700 mb-2",children:"Team-specific"}),d.length>0?(0,t.jsx)("div",{className:"flex flex-wrap gap-2",children:d.map(e=>(0,t.jsx)(D.Tag,{color:"blue",children:e},e))}):(0,t.jsx)("span",{className:"block text-sm text-gray-500",children:"None configured"})]})]}):(0,t.jsx)("span",{className:"block text-gray-500",children:"No guardrails configured"});return"card"===s?(0,t.jsxs)("div",{className:`bg-white border border-gray-200 rounded-lg p-6 ${i}`,children:[(0,t.jsx)("div",{className:"flex items-center gap-2 mb-6",children:(0,t.jsxs)("div",{children:[(0,t.jsx)("span",{className:"block font-semibold text-gray-900",children:"Guardrails Settings"}),(0,t.jsx)("span",{className:"block text-xs text-gray-500",children:"Global and team-specific guardrails applied to this team"})]})}),m]}):(0,t.jsxs)("div",{className:`${i}`,children:[(0,t.jsx)("span",{className:"block font-medium text-gray-900 mb-3",children:"Guardrails Settings"}),m]})};var Y=e.i(643449),J=e.i(75921),X=e.i(390605),Z=e.i(162386),ee=e.i(727749),et=e.i(384767),el=e.i(435451),ea=e.i(916940);let er=({onChange:e,value:l,className:a,accessToken:r,placeholder:s="Select search tools (optional)",disabled:n=!1})=>{let[o,d]=(0,V.useState)([]),[m,c]=(0,V.useState)(!1);return(0,V.useEffect)(()=>{(async()=>{if(r){c(!0);try{let e=await (0,i.fetchSearchTools)(r),t=Array.isArray(e?.search_tools)?e.search_tools:Array.isArray(e?.data)?e.data:[];d(t.map(e=>e?.search_tool_name).filter(e=>"string"==typeof e&&e.length>0).map(e=>({label:e,value:e})))}catch(e){console.error("Failed to load search tools:",e)}finally{c(!1)}}})()},[r]),(0,t.jsx)(O.Select,{mode:"multiple",allowClear:!0,showSearch:!0,optionFilterProp:"label",placeholder:s,onChange:e,value:l,loading:m,className:a,options:o,style:{width:"100%"},disabled:n})};e.s(["default",0,er],471145);var es=e.i(183588),ei=e.i(460285),en=e.i(276173),eo=e.i(91979),ed=e.i(269200),em=e.i(942232),ec=e.i(977572),eu=e.i(427612),eg=e.i(64848),eh=e.i(496020),ep=e.i(536916),ex=e.i(21548);let eb={"/key/generate":"Member can generate a virtual key for this team","/key/service-account/generate":"Member can generate a service account key (not belonging to any user) for this team","/key/update":"Member can update a virtual key belonging to this team","/key/delete":"Member can delete a virtual key belonging to this team","/key/info":"Member can get info about a virtual key belonging to this team","/key/regenerate":"Member can regenerate a virtual key belonging to this team","/key/{key_id}/regenerate":"Member can regenerate a virtual key belonging to this team","/key/list":"Member can list virtual keys belonging to this team","/key/block":"Member can block a virtual key belonging to this team","/key/unblock":"Member can unblock a virtual key belonging to this team","/team/daily/activity":"Member can view all team usage data (not just their own)","/spend/logs":"Member can view spend logs for the entire team (not just their own)"},e_=({teamId:e,accessToken:l,canEditTeam:a})=>{let[r,s]=(0,V.useState)([]),[n,o]=(0,V.useState)([]),[d,m]=(0,V.useState)(!0),[c,u]=(0,V.useState)(!1),[g,h]=(0,V.useState)(!1),p=async()=>{try{if(m(!0),!l)return;let t=await (0,i.getTeamPermissionsCall)(l,e),a=t.all_available_permissions||[];s(a);let r=t.team_member_permissions||[];o(r),h(!1)}catch(e){ee.default.fromBackend("Failed to load permissions"),console.error("Error fetching permissions:",e)}finally{m(!1)}};(0,V.useEffect)(()=>{p()},[e,l]);let x=async()=>{try{if(!l)return;u(!0),await (0,i.teamPermissionsUpdateCall)(l,e,n),ee.default.success("Permissions updated successfully"),h(!1)}catch(e){ee.default.fromBackend("Failed to update permissions"),console.error("Error updating permissions:",e)}finally{u(!1)}};if(d)return(0,t.jsx)("div",{className:"p-6 text-center",children:"Loading permissions..."});let b=r.length>0;return(0,t.jsxs)(C.Card,{className:"bg-white shadow-md rounded-md p-6",children:[(0,t.jsxs)("div",{className:"flex flex-col sm:flex-row justify-between items-start sm:items-center border-b pb-4 mb-6",children:[(0,t.jsx)(k.Title,{className:"mb-2 sm:mb-0",children:"Member Permissions"}),a&&g&&(0,t.jsxs)("div",{className:"flex gap-3",children:[(0,t.jsx)(I.Button,{icon:(0,t.jsx)(eo.ReloadOutlined,{}),onClick:()=>{p()},children:"Reset"}),(0,t.jsx)(I.Button,{onClick:x,loading:c,type:"primary",icon:(0,t.jsx)(_.SaveOutlined,{}),children:"Save Changes"})]})]}),(0,t.jsx)(T.Text,{className:"mb-6 text-gray-600",children:"Control what team members can do when they are not team admins."}),b?(0,t.jsx)("div",{className:"overflow-x-auto",children:(0,t.jsxs)(ed.Table,{className:" min-w-full",children:[(0,t.jsx)(eu.TableHead,{children:(0,t.jsxs)(eh.TableRow,{children:[(0,t.jsx)(eg.TableHeaderCell,{children:"Method"}),(0,t.jsx)(eg.TableHeaderCell,{children:"Endpoint"}),(0,t.jsx)(eg.TableHeaderCell,{children:"Description"}),(0,t.jsx)(eg.TableHeaderCell,{className:"sticky right-0 bg-white shadow-[-4px_0_4px_-4px_rgba(0,0,0,0.1)] text-center",children:"Allow Access"})]})}),(0,t.jsx)(em.TableBody,{children:r.map(e=>{let l=(e=>{let t=e.includes("/info")||e.includes("/list")||e.includes("/activity")||"/spend/logs"===e?"GET":"POST",l=eb[e];if(!l){for(let[t,a]of Object.entries(eb))if(e.includes(t)){l=a;break}}return l||(l=`Access ${e}`),{method:t,endpoint:e,description:l,route:e}})(e);return(0,t.jsxs)(eh.TableRow,{className:"hover:bg-gray-50 transition-colors",children:[(0,t.jsx)(ec.TableCell,{children:(0,t.jsx)("span",{className:`px-2 py-1 rounded text-xs font-medium ${"GET"===l.method?"bg-blue-100 text-blue-800":"bg-green-100 text-green-800"}`,children:l.method})}),(0,t.jsx)(ec.TableCell,{children:(0,t.jsx)("span",{className:"font-mono text-sm text-gray-800",children:l.endpoint})}),(0,t.jsx)(ec.TableCell,{className:"text-gray-700",children:l.description}),(0,t.jsx)(ec.TableCell,{className:"sticky right-0 bg-white shadow-[-4px_0_4px_-4px_rgba(0,0,0,0.1)] text-center",children:(0,t.jsx)(ep.Checkbox,{checked:n.includes(e),onChange:t=>{o(t.target.checked?[...n,e]:n.filter(t=>t!==e)),h(!0)},disabled:!a})})]},e)})})]})}):(0,t.jsx)("div",{className:"py-12",children:(0,t.jsx)(ex.Empty,{description:"No permissions available"})})]})};var ef=e.i(822315);function ey(e){if(!e)return null;let t=(0,ef.default)(e);return t.isValid()?t.format("MMM D, YYYY"):null}var ej=e.i(175712),ev=e.i(178654),ew=e.i(621192),eC=e.i(898586);let eS=async(e,t)=>{let l=(0,i.getProxyBaseUrl)(),a=l?`${l}/team/${encodeURIComponent(t)}/members/me`:`/team/${encodeURIComponent(t)}/members/me`,r=await fetch(a,{method:"GET",headers:{[(0,i.getGlobalLitellmHeaderName)()]:`Bearer ${e}`,"Content-Type":"application/json"}});if(404===r.status)return null;if(!r.ok){let e=await r.json().catch(()=>({}));throw Error((0,i.deriveErrorMessage)(e))}return await r.json()},eT=(e,l)=>(0,t.jsxs)(z.Space,{size:4,children:[(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:e}),(0,t.jsx)(R.Tooltip,{title:l,children:(0,t.jsx)(p.InfoCircleOutlined,{style:{color:"#8c8c8c"}})})]}),eN=(e,t=4)=>null==e?"0":(0,m.formatNumberWithCommas)(e,t),ek=e=>null==e?"Unlimited":(0,m.formatNumberWithCommas)(e,0);function eI({teamId:e}){let{data:a,isLoading:r,error:s}=(e=>{let{accessToken:t}=(0,l.default)();return(0,n.useQuery)({queryKey:["team",e,"members","me"],queryFn:()=>eS(t,e),enabled:!!(t&&e)})})(e);if(r)return(0,t.jsx)(ej.Card,{children:(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"Loading your membership info…"})});if(s)return(0,t.jsx)(ej.Card,{children:(0,t.jsx)(eC.Typography.Text,{type:"danger",children:s instanceof Error?s.message:"Failed to load your membership info for this team."})});if(!a)return(0,t.jsx)(ej.Card,{children:(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"No membership info available for the current user in this team."})});let i=a.litellm_budget_table??null,o=i?.max_budget??null,d=a.spend??0,m=a.total_spend??0,c=i?.tpm_limit??null,u=i?.rpm_limit??null,g=ey(i?.budget_reset_at),h=i?.allowed_models??null;return(0,t.jsxs)(z.Space,{direction:"vertical",size:"middle",style:{width:"100%"},children:[(0,t.jsx)(ej.Card,{children:(0,t.jsxs)(ew.Row,{gutter:[24,16],children:[(0,t.jsxs)(ev.Col,{xs:24,sm:12,md:8,children:[(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"User"}),(0,t.jsx)("div",{style:{marginTop:4},children:(0,t.jsx)(eC.Typography.Text,{strong:!0,children:a.user_email||a.user_id})}),(0,t.jsx)(eC.Typography.Text,{type:"secondary",style:{fontSize:12,fontFamily:"monospace"},children:a.user_id})]}),(0,t.jsxs)(ev.Col,{xs:24,sm:12,md:8,children:[(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"Team Role"}),(0,t.jsx)("div",{style:{marginTop:4},children:(0,t.jsx)(D.Tag,{color:"admin"===a.role?"blue":"default",children:a.role||"user"})})]})]})}),(0,t.jsxs)(ew.Row,{gutter:[16,16],children:[(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Current Cycle Spend (USD)","Spend for the current budget cycle. Resets to $0 when the budget window rolls over."),(0,t.jsxs)("div",{style:{marginTop:8},children:[(0,t.jsxs)(eC.Typography.Title,{level:3,style:{margin:0},children:["$",eN(d,4)]}),(0,t.jsxs)(eC.Typography.Text,{type:"secondary",children:["of ",null===o?"Unlimited":`$${eN(o,4)}`]})]}),g&&(0,t.jsx)("div",{style:{marginTop:4},children:(0,t.jsxs)(eC.Typography.Text,{type:"secondary",children:["Resets ",g]})})]})}),(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Rate Limits","Your per-member rate limits within this team."),(0,t.jsxs)("div",{style:{marginTop:8},children:[(0,t.jsxs)(eC.Typography.Text,{children:["TPM: ",ek(c)]}),(0,t.jsx)("br",{}),(0,t.jsxs)(eC.Typography.Text,{children:["RPM: ",ek(u)]})]})]})}),(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Total Spend (USD)","Cumulative spend across all budget cycles within this team."),(0,t.jsx)("div",{style:{marginTop:8},children:(0,t.jsxs)(eC.Typography.Title,{level:4,style:{margin:0},children:["$",eN(m,4)]})})]})}),(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Model Scope","Models you can access within this team."),(0,t.jsx)("div",{style:{marginTop:8},children:h&&h.length>0?(0,t.jsx)(z.Space,{wrap:!0,children:h.map(e=>(0,t.jsx)(D.Tag,{children:e},e))}):(0,t.jsx)(eC.Typography.Text,{children:"All Team Models"})})]})})]})]})}let eM="overview",eA="my-user",eF="virtual-keys",eO="members",ez="member-permissions",eP="settings",eL={[eM]:"Overview",[eA]:"My User",[eF]:"Virtual Keys",[eO]:"Members",[ez]:"Member Permissions",[eP]:"Settings"};var eD=e.i(292639),eR=e.i(294612);function eE({teamData:e,canEditTeam:a,handleMemberDelete:r,setSelectedEditMember:s,setIsEditMemberModalVisible:i,setIsAddMemberModalVisible:n}){let o=e=>{if(null==e)return"0";if("number"==typeof e){let t=Number(e);return t===Math.floor(t)?t.toString():(0,m.formatNumberWithCommas)(t,8).replace(/\.?0+$/,"")}return"0"},{data:d}=(0,eD.useUISettings)(),{userId:c,userRole:g}=(0,l.default)(),h=!!d?.values?.disable_team_admin_delete_team_user,x=(0,u.isUserTeamAdminForSingleTeam)(e.team_info.members_with_roles,c||""),b=(0,u.isProxyAdminRole)(g||""),_=[{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Model Scope",(0,t.jsx)(R.Tooltip,{title:"Models this member can access. Empty means they inherit all team models.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"model_scope",render:(l,a)=>{let r=(t=>{if(!t)return null;let l=e.team_memberships.find(e=>e.user_id===t),a=l?.litellm_budget_table?.allowed_models;return a&&a.length>0?a:null})(a.user_id);if(!r)return(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"(all team models)"});let s=r.slice(0,2),i=r.length-s.length;return(0,t.jsxs)(z.Space,{wrap:!0,children:[s.map(e=>(0,t.jsx)(eC.Typography.Text,{code:!0,style:{fontSize:"12px"},children:e},e)),i>0&&(0,t.jsx)(R.Tooltip,{title:r.slice(2).join(", "),children:(0,t.jsxs)(eC.Typography.Text,{type:"secondary",children:["+",i," more"]})})]})}},{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Current Cycle Spend (USD)",(0,t.jsx)(R.Tooltip,{title:"Spend for the current budget cycle. Resets to $0 when the member's budget window rolls over. This is the value checked against the member's budget.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"spend",render:(l,a)=>(0,t.jsxs)(eC.Typography.Text,{children:["$",(0,m.formatNumberWithCommas)((t=>{if(!t)return 0;let l=e.team_memberships.find(e=>e.user_id===t);return l?.spend??0})(a.user_id),4)]})},{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Total Spend (USD)",(0,t.jsx)(R.Tooltip,{title:"Cumulative spend by this member within this team, across all budget cycles. Tracking began 2026-04-21; spend from before that date is not included.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"total_spend",render:(l,a)=>(0,t.jsxs)(eC.Typography.Text,{children:["$",(0,m.formatNumberWithCommas)((t=>{if(!t)return 0;let l=e.team_memberships.find(e=>e.user_id===t);return l?.total_spend??0})(a.user_id),4)]})},{title:"Team Member Budget (USD)",key:"budget",render:(l,a)=>{let r=(t=>{if(!t)return null;let l=e.team_memberships.find(e=>e.user_id===t),a=l?.litellm_budget_table?.max_budget;return null==a?null:o(a)})(a.user_id);return(0,t.jsx)(eC.Typography.Text,{children:r?`$${(0,m.formatNumberWithCommas)(Number(r),4)}`:"No Limit"})}},{title:"Budget Reset",key:"budget_reset",render:(l,a)=>{let r=(t=>{if(!t)return null;let l=e.team_memberships.find(e=>e.user_id===t);return ey(l?.litellm_budget_table?.budget_reset_at)})(a.user_id);return r?(0,t.jsx)(eC.Typography.Text,{children:r}):(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"—"})}},{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Team Member Rate Limits",(0,t.jsx)(R.Tooltip,{title:"Rate limits for this member's usage within this team.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"rate_limits",render:(l,a)=>(0,t.jsx)(eC.Typography.Text,{children:(t=>{if(!t)return"No Limits";let l=e.team_memberships.find(e=>e.user_id===t),a=l?.litellm_budget_table?.rpm_limit,r=l?.litellm_budget_table?.tpm_limit,s=[a?`${o(a)} RPM`:null,r?`${o(r)} TPM`:null].filter(Boolean);return s.length>0?s.join(" / "):"No Limits"})(a.user_id)})}];return(0,t.jsx)(eR.default,{members:e.team_info.members_with_roles,canEdit:a,onEdit:t=>{let l=e.team_memberships.find(e=>e.user_id===t.user_id);s({...t,max_budget_in_team:l?.litellm_budget_table?.max_budget||null,tpm_limit:l?.litellm_budget_table?.tpm_limit||null,rpm_limit:l?.litellm_budget_table?.rpm_limit||null,allowed_models:l?.litellm_budget_table?.allowed_models||[]}),i(!0)},onDelete:r,onAddMember:()=>n(!0),roleColumnTitle:"Team Role",roleTooltip:"This role applies only to this team and is independent from the user's proxy-level role.",extraColumns:_,showDeleteForMember:()=>b||a&&!x||x&&!h})}var eB=e.i(207082),eU=e.i(871943),eV=e.i(502547),eK=e.i(360820),e$=e.i(94629),eG=e.i(152990),eW=e.i(682830),eq=e.i(994388),eH=e.i(752978),eQ=e.i(282786),eY=e.i(981339),eJ=e.i(304911),eX=e.i(969550),eZ=e.i(20147),e0=e.i(633627);function e1({teamId:e,teamAlias:a,organization:r}){let{accessToken:s}=(0,l.default)(),[i,o]=(0,V.useState)(null),[d,c]=(0,V.useState)([{id:"created_at",desc:!0}]),[u,g]=(0,V.useState)({pageIndex:0,pageSize:50}),[h,x]=(0,V.useState)({"Organization ID":"","Key Alias":"","User ID":"","Sort By":"created_at","Sort Order":"desc"}),b=d.length>0?d[0].id:"created_at",_=d.length>0?d[0].desc?"desc":"asc":"desc",f=u.pageIndex,y=u.pageSize,{data:j,isPending:v,isFetching:C,refetch:S}=(0,eB.useKeys)(f+1,y,{teamID:e,organizationID:h["Organization ID"]?.trim()||void 0,selectedKeyAlias:h["Key Alias"]?.trim()||void 0,userID:h["User ID"]?.trim()||void 0,sortBy:b||void 0,sortOrder:_||void 0,expand:"user"}),N=(0,V.useMemo)(()=>{let e=j?.keys||[],t=r?.organization_id;return t?e.map(e=>({...e,organization_id:(e.organization_id??e.org_id)||t})):e},[j?.keys,r?.organization_id]),k=j?.total_pages??0,[I,M]=(0,V.useState)({}),A=(0,V.useMemo)(()=>({team_id:e,team_alias:a||e,models:[],max_budget:null,budget_duration:null,tpm_limit:null,rpm_limit:null,organization_id:r?.organization_id||"",created_at:"",keys:[],members_with_roles:[],spend:0}),[e,a,r]),F=(0,n.useQuery)({queryKey:["teamFilterOptions",e,s],queryFn:async()=>(0,e0.fetchTeamFilterOptions)(s,e),enabled:!!s&&!!e,staleTime:3e4}).data||{keyAliases:[],organizationIds:[],userIds:[]},O=(0,V.useCallback)(()=>{S?.()},[S]);(0,V.useEffect)(()=>(window.addEventListener("storage",O),()=>window.removeEventListener("storage",O)),[O]);let z=(0,V.useCallback)((e,t=!1)=>{x(t=>({...t,"Organization ID":e["Organization ID"]??t["Organization ID"],"Key Alias":e["Key Alias"]??t["Key Alias"],"User ID":e["User ID"]??t["User ID"],"Sort By":e["Sort By"]??t["Sort By"]??"created_at","Sort Order":e["Sort Order"]??t["Sort Order"]??"desc"})),t||g(e=>({...e,pageIndex:0}))},[]),P=(0,V.useCallback)(()=>{x({"Organization ID":"","Key Alias":"","User ID":"","Sort By":"created_at","Sort Order":"desc"}),g(e=>({...e,pageIndex:0}))},[]),L=(0,V.useMemo)(()=>[{name:"Organization ID",label:"Organization ID",isSearchable:!0,searchFn:async e=>{let{organizationIds:t}=F;if(!t.length)return[];let l=e.toLowerCase();return(l?t.filter(e=>e.toLowerCase().includes(l)):t).map(e=>({label:e,value:e}))}},{name:"Key Alias",label:"Key Alias",isSearchable:!0,searchFn:async e=>{let{keyAliases:t}=F,l=e.toLowerCase();return(l?t.filter(e=>e.toLowerCase().includes(l)):t).map(e=>({label:e,value:e}))}},{name:"User ID",label:"User ID",isSearchable:!0,searchFn:async e=>{let{userIds:t}=F,l=e.toLowerCase();return(l?t.filter(e=>e.id.toLowerCase().includes(l)||e.email.toLowerCase().includes(l)):t).map(e=>({label:e.email?`${e.id} (${e.email})`:e.id,value:e.id}))}}],[F]),D=(0,V.useMemo)(()=>[{id:"token",accessorKey:"token",header:"Key ID",size:100,enableSorting:!0,cell:e=>{let l=e.getValue(),a=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:l,children:(0,t.jsx)(eq.Button,{size:"xs",variant:"light",className:"font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate block",style:{maxWidth:a,overflow:"hidden"},onClick:()=>o(e.row.original),children:l??"-"})})}},{id:"key_alias",accessorKey:"key_alias",header:"Key Alias",size:150,enableSorting:!0,cell:e=>{let l=e.getValue(),a=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:l,children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block",style:{maxWidth:a,overflow:"hidden"},children:l??"-"})})}},{id:"key_name",accessorKey:"key_name",header:"Secret Key",size:120,enableSorting:!1,cell:e=>(0,t.jsx)("span",{className:"font-mono text-xs",children:e.getValue()})},{id:"organization_id",accessorKey:"organization_id",header:"Organization ID",size:140,enableSorting:!1,cell:e=>e.getValue()?e.renderValue():"-"},{id:"user_email",accessorKey:"user",header:"User Email",size:160,enableSorting:!1,cell:e=>{let l=e.getValue(),a=l?.user_email,r=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:a,children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block",style:{maxWidth:r,overflow:"hidden"},children:a??"-"})})}},{id:"user_id",accessorKey:"user_id",header:"User ID",size:70,enableSorting:!1,cell:e=>{let l=e.getValue(),a="default_user_id"===l?"Default Proxy Admin":l,r=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:a,children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block",style:{maxWidth:r,overflow:"hidden"},children:a??"-"})})}},{id:"created_at",accessorKey:"created_at",header:"Created At",size:120,enableSorting:!0,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleDateString():"-"}},{id:"created_by",accessorKey:"created_by",header:"Created By",size:70,enableSorting:!1,cell:e=>{let l=e.getValue();if(!l)return"-";let{created_by_user:a}=e.row.original,r=a?.user_alias??null,s=a?.user_email??null,i="default_user_id"===l,n=r||s||l,o=e.cell.column.getSize(),d=(0,t.jsx)("div",{className:"flex flex-col gap-2 text-xs min-w-[200px] max-w-[300px]",children:[{label:"User Alias",value:r},{label:"User Email",value:s},{label:"User ID",value:l}].map(({label:e,value:l})=>(0,t.jsxs)("div",{className:"flex flex-col min-w-0",children:[(0,t.jsx)("span",{className:"text-gray-400",children:e}),l?(0,t.jsx)(eC.Typography.Text,{className:"font-mono text-xs",ellipsis:{tooltip:l},copyable:!0,children:l}):(0,t.jsx)("span",{className:"font-mono",children:"-"})]},e))});return!i||r||s?(0,t.jsx)(eQ.Popover,{content:d,trigger:"hover",placement:"bottomLeft",children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block cursor-default",style:{maxWidth:o,overflow:"hidden"},children:n})}):(0,t.jsx)(eQ.Popover,{content:d,trigger:"hover",placement:"bottomLeft",children:(0,t.jsx)("span",{className:"cursor-default",children:(0,t.jsx)(eJ.default,{userId:l})})})}},{id:"updated_at",accessorKey:"updated_at",header:"Updated At",size:120,enableSorting:!0,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleDateString():"Never"}},{id:"last_active",accessorKey:"last_active",header:()=>(0,t.jsxs)("span",{className:"flex items-center gap-1",children:["Last Active",(0,t.jsx)(eQ.Popover,{content:"This is a new field and is not backfilled. Only new key usage will update this value.",trigger:"hover",children:(0,t.jsx)(p.InfoCircleOutlined,{className:"text-gray-400 text-xs cursor-help"})})]}),size:130,enableSorting:!1,cell:e=>{let l=e.getValue();if(!l)return"Unknown";let a=new Date(l);return(0,t.jsx)(R.Tooltip,{title:a.toLocaleString(void 0,{dateStyle:"medium",timeStyle:"long"}),children:(0,t.jsx)("span",{children:a.toLocaleDateString()})})}},{id:"expires",accessorKey:"expires",header:"Expires",size:120,enableSorting:!1,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleDateString():"Never"}},{id:"spend",accessorKey:"spend",header:"Spend (USD)",size:100,enableSorting:!0,cell:e=>(0,m.formatNumberWithCommas)(e.getValue(),4)},{id:"max_budget",accessorKey:"max_budget",header:"Budget (USD)",size:110,enableSorting:!0,cell:e=>{let t=e.getValue();return null===t?"Unlimited":`$${(0,m.formatNumberWithCommas)(t)}`}},{id:"budget_reset_at",accessorKey:"budget_reset_at",header:"Budget Reset",size:130,enableSorting:!1,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleString():"Never"}},{id:"models",accessorKey:"models",header:"Models",size:200,enableSorting:!1,cell:e=>{let l=e.getValue();return(0,t.jsx)("div",{className:"flex flex-col py-2",children:Array.isArray(l)?(0,t.jsx)("div",{className:"flex flex-col",children:0===l.length?(0,t.jsx)(w.Badge,{size:"xs",className:"mb-1",color:"red",children:(0,t.jsx)(T.Text,{children:"All Proxy Models"})}):(0,t.jsx)(t.Fragment,{children:(0,t.jsxs)("div",{className:"flex items-start",children:[l.length>3&&(0,t.jsx)("div",{children:(0,t.jsx)(eH.Icon,{icon:I[e.row.id]?eU.ChevronDownIcon:eV.ChevronRightIcon,className:"cursor-pointer",size:"xs",onClick:()=>M(t=>({...t,[e.row.id]:!t[e.row.id]}))})}),(0,t.jsxs)("div",{className:"flex flex-wrap gap-1",children:[l.slice(0,3).map((e,l)=>"all-proxy-models"===e?(0,t.jsx)(w.Badge,{size:"xs",color:"red",children:(0,t.jsx)(T.Text,{children:"All Proxy Models"})},l):(0,t.jsx)(w.Badge,{size:"xs",color:"blue",children:(0,t.jsx)(T.Text,{children:e.length>30?`${(0,H.getModelDisplayName)(e).slice(0,30)}...`:(0,H.getModelDisplayName)(e)})},l)),l.length>3&&!I[e.row.id]&&(0,t.jsx)(w.Badge,{size:"xs",color:"gray",className:"cursor-pointer",children:(0,t.jsxs)(T.Text,{children:["+",l.length-3," ",l.length-3==1?"more model":"more models"]})}),I[e.row.id]&&(0,t.jsx)("div",{className:"flex flex-wrap gap-1",children:l.slice(3).map((e,l)=>"all-proxy-models"===e?(0,t.jsx)(w.Badge,{size:"xs",color:"red",children:(0,t.jsx)(T.Text,{children:"All Proxy Models"})},l+3):(0,t.jsx)(w.Badge,{size:"xs",color:"blue",children:(0,t.jsx)(T.Text,{children:e.length>30?`${(0,H.getModelDisplayName)(e).slice(0,30)}...`:(0,H.getModelDisplayName)(e)})},l+3))})]})]})})}):null})}},{id:"rate_limits",header:"Rate Limits",size:140,enableSorting:!1,cell:({row:e})=>{let l=e.original;return(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{children:["TPM: ",null!==l.tpm_limit?l.tpm_limit:"Unlimited"]}),(0,t.jsxs)("div",{children:["RPM: ",null!==l.rpm_limit?l.rpm_limit:"Unlimited"]})]})}}],[I]),E=(0,V.useCallback)(e=>{let t="function"==typeof e?e(d):e;if(c(t),t?.length>0){let e=t[0];z({"Sort By":e.id,"Sort Order":e.desc?"desc":"asc"},!0)}},[d,z]),B=(0,eG.useReactTable)({data:N,columns:D,columnResizeMode:"onChange",columnResizeDirection:"ltr",state:{sorting:d,pagination:u},onSortingChange:E,onPaginationChange:g,getCoreRowModel:(0,eW.getCoreRowModel)(),enableSorting:!0,manualSorting:!0,manualPagination:!0,pageCount:k});return(0,t.jsx)("div",{className:"w-full h-full overflow-hidden",children:i?(0,t.jsx)(eZ.default,{keyId:i.token,onClose:()=>o(null),keyData:i,teams:[A],onDelete:S}):(0,t.jsxs)("div",{className:"border-b py-4 flex-1 overflow-hidden",children:[(0,t.jsx)("div",{className:"w-full mb-6",children:(0,t.jsx)(eX.default,{options:L,onApplyFilters:z,initialValues:h,onResetFilters:P})}),(0,t.jsx)("div",{className:"flex items-center justify-end w-full mb-4",children:(0,t.jsxs)("div",{className:"inline-flex items-center gap-2",children:[v||C?(0,t.jsx)(eY.Skeleton.Node,{active:!0,style:{width:74,height:20}}):(0,t.jsxs)("span",{className:"text-sm text-gray-700",children:["Page ",f+1," of ",B.getPageCount()]}),v||C?(0,t.jsx)(eY.Skeleton.Button,{active:!0,size:"small",style:{width:84,height:30}}):(0,t.jsx)("button",{onClick:()=>B.previousPage(),disabled:v||C||!B.getCanPreviousPage(),className:"px-3 py-1 text-sm border rounded-md hover:bg-gray-50 disabled:opacity-50 disabled:cursor-not-allowed",children:"Previous"}),v||C?(0,t.jsx)(eY.Skeleton.Button,{active:!0,size:"small",style:{width:58,height:30}}):(0,t.jsx)("button",{onClick:()=>B.nextPage(),disabled:v||C||!B.getCanNextPage(),className:"px-3 py-1 text-sm border rounded-md hover:bg-gray-50 disabled:opacity-50 disabled:cursor-not-allowed",children:"Next"})]})}),(0,t.jsx)("div",{className:"h-[75vh] overflow-auto",children:(0,t.jsx)("div",{className:"rounded-lg custom-border relative",children:(0,t.jsx)("div",{className:"overflow-x-auto",children:(0,t.jsxs)(ed.Table,{className:"[&_td]:py-0.5 [&_th]:py-1",style:{width:B.getCenterTotalSize()},children:[(0,t.jsx)(eu.TableHead,{children:B.getHeaderGroups().map(e=>(0,t.jsx)(eh.TableRow,{children:e.headers.map(e=>(0,t.jsx)(eg.TableHeaderCell,{"data-header-id":e.id,className:`py-1 h-8 relative hover:bg-gray-50 ${"actions"===e.id?"sticky right-0 bg-white shadow-[-4px_0_8px_-6px_rgba(0,0,0,0.1)]":""}`,style:{width:e.getSize(),position:"relative",cursor:e.column.getCanSort()?"pointer":"default"},onMouseEnter:()=>{let t=document.querySelector(`[data-header-id="${e.id}"] .resizer`);t&&(t.style.opacity="0.5")},onMouseLeave:()=>{let t=document.querySelector(`[data-header-id="${e.id}"] .resizer`);t&&!e.column.getIsResizing()&&(t.style.opacity="0")},onClick:e.column.getCanSort()?e.column.getToggleSortingHandler():void 0,children:(0,t.jsxs)("div",{className:"flex items-center justify-between gap-2",children:[(0,t.jsx)("div",{className:"flex items-center",children:e.isPlaceholder?null:(0,eG.flexRender)(e.column.columnDef.header,e.getContext())}),"actions"!==e.id&&e.column.getCanSort()&&(0,t.jsx)("div",{className:"w-4",children:e.column.getIsSorted()?({asc:(0,t.jsx)(eK.ChevronUpIcon,{className:"h-4 w-4 text-blue-500"}),desc:(0,t.jsx)(eU.ChevronDownIcon,{className:"h-4 w-4 text-blue-500"})})[e.column.getIsSorted()]:(0,t.jsx)(e$.SwitchVerticalIcon,{className:"h-4 w-4 text-gray-400"})}),(0,t.jsx)("div",{onDoubleClick:()=>e.column.resetSize(),onMouseDown:e.getResizeHandler(),onTouchStart:e.getResizeHandler(),className:`resizer ${B.options.columnResizeDirection} ${e.column.getIsResizing()?"isResizing":""}`,style:{position:"absolute",right:0,top:0,height:"100%",width:"5px",background:e.column.getIsResizing()?"#3b82f6":"transparent",cursor:"col-resize",userSelect:"none",touchAction:"none",opacity:+!!e.column.getIsResizing()}})]})},e.id))},e.id))}),(0,t.jsx)(em.TableBody,{children:v||C?(0,t.jsx)(eh.TableRow,{children:(0,t.jsx)(ec.TableCell,{colSpan:D.length,className:"h-8 text-center",children:(0,t.jsx)("div",{className:"text-center text-gray-500",children:(0,t.jsx)("p",{children:"Loading keys..."})})})}):N.length>0?B.getRowModel().rows.map(e=>(0,t.jsx)(eh.TableRow,{className:"h-8",children:e.getVisibleCells().map(e=>(0,t.jsx)(ec.TableCell,{style:{width:e.column.getSize(),maxWidth:"8-x",whiteSpace:"pre-wrap",overflow:"hidden"},className:`py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap ${"models"===e.column.id&&Array.isArray(e.getValue())&&e.getValue().length>3?"px-0":""}`,children:(0,eG.flexRender)(e.column.columnDef.cell,e.getContext())},e.id))},e.id)):(0,t.jsx)(eh.TableRow,{children:(0,t.jsx)(ec.TableCell,{colSpan:D.length,className:"h-8 text-center",children:(0,t.jsx)("div",{className:"text-center text-gray-500",children:(0,t.jsx)("p",{children:"No keys found"})})})})})]})})})})]})})}e.s(["default",0,({teamId:e,onClose:n,accessToken:o,is_team_admin:eo,is_proxy_admin:ed,is_org_admin:em=!1,userModels:ec,editTeam:eu,premiumUser:eg=!1,onUpdate:eh})=>{let ep,ex,eb,ef,ey,ej,[ev,ew]=(0,V.useState)(null),[eC,eS]=(0,V.useState)(!0),[eT,eN]=(0,V.useState)(!1),[ek]=M.Form.useForm(),[eD,eR]=(0,V.useState)(!1),[eB,eU]=(0,V.useState)(null),[eV,eK]=(0,V.useState)(!1),[e$,eG]=(0,V.useState)([]),[eW,eq]=(0,V.useState)(!1),[eH,eQ]=(0,V.useState)({}),{data:eY,isLoading:eJ}=d(),eX=eY?.globalGuardrailNames??new Set,[eZ,e0]=(0,V.useState)([]),[e2,e4]=(0,V.useState)({}),[e5,e3]=(0,V.useState)(!1),[e7,e6]=(0,V.useState)(null),[e9,e8]=(0,V.useState)(!1),[te,tt]=(0,V.useState)(!1),[tl,ta]=(0,V.useState)(!1),tr=V.default.useRef(null),[ts,ti]=(0,V.useState)(null),{userRole:tn,userId:to}=(0,l.default)(),{data:td=[]}=(0,a.useOrganizations)(),tm=(0,r.useQueryClient)(),tc=(0,V.useMemo)(()=>{let e=ev?.team_info?.organization_id;if(!e||!to)return!1;let t=td.find(t=>t.organization_id===e);return t?.members?.some(e=>e.user_id===to&&"org_admin"===e.user_role)??!1},[ev,td,to]),tu=M.Form.useWatch("models",ek),tg=M.Form.useWatch("disable_global_guardrails",ek),th=(0,V.useMemo)(()=>{let e=tu??ev?.team_info?.models??[];return e.includes("all-proxy-models")||e.includes("all-team-models")?ec:(0,H.unfurlWildcardModelsInList)(e,ec)},[tu,ev,ec]),tp=eo||ed||em||tc,tx=(0,V.useMemo)(()=>{let e;return e=[eM,eA,eF],tp?[...e,eO,ez,eP]:e},[tp]),tb=(0,V.useMemo)(()=>eu&&tp?eP:eM,[eu,tp]),t_=async()=>{try{if(eS(!0),!o)return;let t=await (0,i.teamInfoCall)(o,e);ew(t)}catch(e){ee.default.fromBackend("Failed to load team information"),console.error("Error fetching team info:",e)}finally{eS(!1)}};(0,V.useEffect)(()=>{t_()},[e,o]),(0,V.useEffect)(()=>{(async()=>{if(!o||!ev?.team_info?.organization_id)return ti(null);try{let e=await (0,i.organizationInfoCall)(o,ev.team_info.organization_id);ti(e)}catch(e){console.error("Error fetching organization info:",e),ti(null)}})()},[o,ev?.team_info?.organization_id]),(0,V.useMemo)(()=>{let e;return e=[],e=ts?ts.models.includes("all-proxy-models")?ec:ts.models.length>0?ts.models:ec:ec,(0,H.unfurlWildcardModelsInList)(e,ec)},[ts,ec]),(0,V.useEffect)(()=>{(async()=>{try{if(!o)return;let e=(await (0,i.getPoliciesList)(o)).policies.map(e=>e.policy_name);e0(e)}catch(e){console.error("Failed to fetch policies:",e)}})()},[o]),(0,V.useEffect)(()=>{(async()=>{if(!o||!ev?.team_info?.policies||0===ev.team_info.policies.length)return;e3(!0);let e={};try{await Promise.all(ev.team_info.policies.map(async t=>{try{let l=await (0,i.getPolicyInfoWithGuardrails)(o,t);e[t]=l.resolved_guardrails||[]}catch(l){console.error(`Failed to fetch guardrails for policy ${t}:`,l),e[t]=[]}})),e4(e)}catch(e){console.error("Failed to fetch policy guardrails:",e)}finally{e3(!1)}})()},[o,ev?.team_info?.policies]);let tf=async t=>{try{if(null==o)return;let l={user_email:t.user_email,user_id:t.user_id,role:t.role};await (0,i.teamMemberAddCall)(o,e,l),ee.default.success("Team member added successfully"),eN(!1),ek.resetFields();let a=await (0,i.teamInfoCall)(o,e);ew(a),eh(a)}catch(t){let e="Failed to add team member";t?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")?e="Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this.":t?.message&&(e=t.message),ee.default.fromBackend(e),console.error("Error adding team member:",t)}},ty=async t=>{try{if(null==o)return;let l={user_email:t.user_email,user_id:t.user_id,role:t.role,max_budget_in_team:t.max_budget_in_team,tpm_limit:t.tpm_limit,rpm_limit:t.rpm_limit,allowed_models:t.allowed_models};E.default.destroy(),await (0,i.teamMemberUpdateCall)(o,e,l),ee.default.success("Team member updated successfully"),eR(!1);let a=await (0,i.teamInfoCall)(o,e);ew(a),eh(a)}catch(t){let e="Failed to update team member";t?.raw?.detail?.includes("Assigning team admins is a premium feature")?e="Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this.":t?.message&&(e=t.message),eR(!1),E.default.destroy(),ee.default.fromBackend(e),console.error("Error updating team member:",t)}},tj=async()=>{if(e7&&o){tt(!0);try{await (0,i.teamMemberDeleteCall)(o,e,e7),ee.default.success("Team member removed successfully");let t=await (0,i.teamInfoCall)(o,e);ew(t),eh(t)}catch(e){ee.default.fromBackend("Failed to remove team member"),console.error("Error removing team member:",e)}finally{tt(!1),e8(!1),e6(null)}}},tv=async t=>{try{let l;if(!o)return;ta(!0);let r={};try{let{soft_budget_alerting_emails:e,...l}=t.metadata?JSON.parse(t.metadata):{};r=l}catch(e){ee.default.fromBackend("Invalid JSON in metadata field");return}if("string"==typeof t.secret_manager_settings&&t.secret_manager_settings.trim().length>0)try{l=JSON.parse(t.secret_manager_settings)}catch(e){ee.default.fromBackend("Invalid JSON in secret manager settings");return}let s=e=>null==e||"string"==typeof e&&""===e.trim()||"number"==typeof e&&Number.isNaN(e)?null:e,n={},d={};for(let e of t.modelLimits??[])e?.model&&(null!=e.tpm&&(n[e.model]=e.tpm),null!=e.rpm&&(d[e.model]=e.rpm));let m=!0===t.disable_global_guardrails,u=m?Array.from(eX):Array.from(eX).filter(e=>!(t.guardrails||[]).includes(e)),g={team_id:e,team_alias:t.team_alias,models:t.models,tpm_limit:s(t.tpm_limit),rpm_limit:s(t.rpm_limit),model_tpm_limit:n,model_rpm_limit:d,max_budget:t.max_budget,soft_budget:s(t.soft_budget),budget_duration:t.budget_duration,metadata:{...r,guardrails:(t.guardrails||[]).filter(e=>!eX.has(e)),opted_out_global_guardrails:u,...t.logging_settings?.length>0?{logging:t.logging_settings}:{},disable_global_guardrails:m,soft_budget_alerting_emails:"string"==typeof t.soft_budget_alerting_emails?t.soft_budget_alerting_emails.split(",").map(e=>e.trim()).filter(e=>e.length>0):t.soft_budget_alerting_emails||[],...void 0!==l?{secret_manager_settings:l}:{}},...t.policies?.length>0?{policies:t.policies}:{},...t.organization_id!==tw.organization_id?{organization_id:t.organization_id??null}:{}};g.max_budget=(0,c.mapEmptyStringToNull)(g.max_budget),g.team_member_budget_duration=t.team_member_budget_duration,void 0!==t.team_member_budget&&(g.team_member_budget=Number(t.team_member_budget)),void 0!==t.team_member_key_duration&&(g.team_member_key_duration=t.team_member_key_duration),(void 0!==t.team_member_tpm_limit||void 0!==t.team_member_rpm_limit)&&(g.team_member_tpm_limit=s(t.team_member_tpm_limit),g.team_member_rpm_limit=s(t.team_member_rpm_limit));let{servers:h,accessGroups:p,toolsets:x}=t.mcp_servers_and_groups||{servers:[],accessGroups:[],toolsets:[]},b=new Set(h||[]),_=Object.fromEntries(Object.entries(t.mcp_tool_permissions||{}).filter(([e])=>b.has(e)));g.object_permission={},h&&(g.object_permission.mcp_servers=h),p&&(g.object_permission.mcp_access_groups=p),_&&(g.object_permission.mcp_tool_permissions=_),x&&(g.object_permission.mcp_toolsets=x),delete t.mcp_servers_and_groups,delete t.mcp_tool_permissions;let{agents:f,accessGroups:y}=t.agents_and_groups||{agents:[],accessGroups:[]};f&&f.length>0&&(g.object_permission.agents=f),y&&y.length>0&&(g.object_permission.agent_access_groups=y),delete t.agents_and_groups,t.vector_stores&&t.vector_stores.length>0&&(g.object_permission.vector_stores=t.vector_stores),Array.isArray(t.object_permission_search_tools)&&(g.object_permission.search_tools=t.object_permission_search_tools),void 0!==t.access_group_ids&&(g.access_group_ids=t.access_group_ids),void 0!==t.default_team_member_models&&(g.default_team_member_models=t.default_team_member_models);let j=tr.current?.getValue();if(j?.router_settings){let e=e=>null!=e&&""!==e&&!1!==e&&!(Array.isArray(e)&&0===e.length),t=Object.values(j.router_settings).some(e),l=tw.router_settings&&Object.values(tw.router_settings).some(e);(t||l)&&(g.router_settings=j.router_settings)}await (0,i.teamUpdateCall)(o,g),tm.invalidateQueries({queryKey:a.organizationKeys.all}),ee.default.success("Team settings updated successfully"),eK(!1),t_()}catch(e){console.error("Error updating team:",e)}finally{ta(!1)}};if(eC)return(0,t.jsx)("div",{className:"p-4",children:"Loading..."});if(!ev?.team_info)return(0,t.jsx)("div",{className:"p-4",children:"Team not found"});let{team_info:tw}=ev,tC=tw.metadata?.disable_global_guardrails===!0,tS=new Set(Array.isArray(tw.metadata?.opted_out_global_guardrails)?tw.metadata.opted_out_global_guardrails:[]),tT=(Array.isArray(tw.metadata?.guardrails)?tw.metadata.guardrails:[]).filter(e=>!eX.has(e)),tN=tC?tT:[...Array.from(eX).filter(e=>!tS.has(e)),...tT],tk=e=>{e.preventDefault(),e.stopPropagation()},tI=async(e,t)=>{await (0,m.copyToClipboard)(e)&&(eQ(e=>({...e,[t]:!0})),setTimeout(()=>{eQ(e=>({...e,[t]:!1}))},2e3))};return(0,t.jsxs)("div",{className:"p-4",children:[(0,t.jsx)("div",{className:"flex justify-between items-center mb-6",children:(0,t.jsxs)("div",{children:[(0,t.jsx)(I.Button,{type:"text",icon:(0,t.jsx)(f.ArrowLeftIcon,{className:"h-4 w-4"}),onClick:n,className:"mb-4",children:"Back to Teams"}),(0,t.jsx)(k.Title,{children:tw.team_alias}),(0,t.jsxs)("div",{className:"flex items-center",children:[(0,t.jsx)(T.Text,{className:"text-gray-500 font-mono",children:tw.team_id}),(0,t.jsx)(I.Button,{type:"text",size:"small",icon:eH["team-id"]?(0,t.jsx)(B.CheckIcon,{size:12}):(0,t.jsx)(U.CopyIcon,{size:12}),onClick:()=>tI(tw.team_id,"team-id"),className:`left-2 z-10 transition-all duration-200 ${eH["team-id"]?"text-green-600 bg-green-50 border-green-200":"text-gray-500 hover:text-gray-700 hover:bg-gray-100"}`})]})]})}),(0,t.jsx)(L.Tabs,{defaultActiveKey:tb,className:"mb-4",items:[{key:eM,label:eL[eM],children:(0,t.jsxs)(S.Grid,{numItems:1,numItemsSm:2,numItemsLg:3,className:"gap-6",children:[(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{children:"Budget Status"}),(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsxs)(k.Title,{children:["$",(0,m.formatNumberWithCommas)(tw.spend,4)]}),(0,t.jsxs)(T.Text,{children:["of ",null===tw.max_budget?"Unlimited":`$${(0,m.formatNumberWithCommas)(tw.max_budget,4)}`]}),tw.budget_duration&&(0,t.jsxs)(T.Text,{className:"text-gray-500",children:["Reset: ",tw.budget_duration]}),(0,t.jsx)("br",{}),tw.team_member_budget_table&&(0,t.jsxs)(T.Text,{className:"text-gray-500",children:["Team Member Budget: $",(0,m.formatNumberWithCommas)(tw.team_member_budget_table.max_budget,4)]})]})]}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{children:"Rate Limits"}),(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsxs)(T.Text,{children:["TPM: ",tw.tpm_limit||"Unlimited"]}),(0,t.jsxs)(T.Text,{children:["RPM: ",tw.rpm_limit||"Unlimited"]}),tw.max_parallel_requests&&(0,t.jsxs)(T.Text,{children:["Max Parallel Requests: ",tw.max_parallel_requests]}),(ep=tw.metadata?.model_tpm_limit??{},ex=tw.metadata?.model_rpm_limit??{},0===(eb=Array.from(new Set([...Object.keys(ep),...Object.keys(ex)]))).length?null:(0,t.jsxs)("div",{className:"mt-3",children:[(0,t.jsx)(T.Text,{className:"text-gray-500",children:"Per-model limits:"}),eb.map(e=>(0,t.jsxs)(T.Text,{className:"text-xs",children:[e,": TPM ",ep[e]??"—",", RPM ",ex[e]??"—"]},e))]}))]})]}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{children:"Models"}),(0,t.jsx)("div",{className:"mt-2 flex flex-wrap gap-2",children:0===tw.models.length||tw.models.includes("all-proxy-models")?(0,t.jsx)(w.Badge,{color:"red",children:"All proxy models"}):(0,t.jsxs)(t.Fragment,{children:[tw.models.map((e,l)=>(0,t.jsx)(w.Badge,{color:"blue",children:e},`direct-${l}`)),(tw.access_group_models||[]).map((e,l)=>(0,t.jsx)(w.Badge,{color:"green",title:"From access group",children:e},`ag-${l}`))]})})]}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{className:"font-semibold text-gray-900",children:"Virtual Keys"}),(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsxs)(T.Text,{children:["User Keys: ",ev.keys.filter(e=>e.user_id).length]}),(0,t.jsxs)(T.Text,{children:["Service Account Keys: ",ev.keys.filter(e=>!e.user_id).length]}),(0,t.jsxs)(T.Text,{className:"text-gray-500",children:["Total: ",ev.keys.length]})]})]}),(0,t.jsx)(et.default,{objectPermission:tw.object_permission,variant:"card",accessToken:o}),(0,t.jsx)(C.Card,{children:(0,t.jsx)(Q,{globalGuardrailNames:eX,teamGuardrails:Array.isArray(tw.metadata?.guardrails)?tw.metadata.guardrails:[],optedOutGlobalGuardrails:Array.isArray(tw.metadata?.opted_out_global_guardrails)?tw.metadata.opted_out_global_guardrails:[],killSwitchOn:tC,variant:"inline"})}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{className:"font-semibold text-gray-900 mb-3",children:"Policies"}),tw.policies&&tw.policies.length>0?(0,t.jsx)("div",{className:"space-y-4",children:tw.policies.map((e,l)=>(0,t.jsxs)("div",{className:"space-y-2",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)(w.Badge,{color:"purple",children:e}),e5&&(0,t.jsx)(T.Text,{className:"text-xs text-gray-400",children:"Loading guardrails..."})]}),!e5&&e2[e]&&e2[e].length>0&&(0,t.jsxs)("div",{className:"ml-4 pl-3 border-l-2 border-gray-200",children:[(0,t.jsx)(T.Text,{className:"text-xs text-gray-500 mb-1",children:"Resolved Guardrails:"}),(0,t.jsx)("div",{className:"flex flex-wrap gap-1",children:e2[e].map((e,l)=>(0,t.jsx)(w.Badge,{color:"blue",size:"xs",children:e},l))})]})]},l))}):(0,t.jsx)(T.Text,{className:"text-gray-500",children:"No policies configured"})]}),(0,t.jsx)(Y.default,{loggingConfigs:tw.metadata?.logging||[],disabledCallbacks:[],variant:"card"})]})},{key:eA,label:eL[eA],children:(0,t.jsx)(eI,{teamId:e})},{key:eF,label:eL[eF],children:(0,t.jsx)(e1,{teamId:e,teamAlias:tw.team_alias,organization:ts})},{key:eO,label:eL[eO],children:(0,t.jsx)(eE,{teamData:ev,canEditTeam:tp,handleMemberDelete:e=>{e6(e),e8(!0)},setSelectedEditMember:eU,setIsEditMemberModalVisible:eR,setIsAddMemberModalVisible:eN})},{key:ez,label:eL[ez],children:(0,t.jsx)(e_,{teamId:e,accessToken:o,canEditTeam:tp})},{key:eP,label:eL[eP],children:(0,t.jsxs)(C.Card,{className:"overflow-y-auto max-h-[65vh]",children:[(0,t.jsxs)("div",{className:"flex justify-between items-center mb-4",children:[(0,t.jsx)(k.Title,{children:"Team Settings"}),tp&&!eV&&(0,t.jsx)(I.Button,{icon:(0,t.jsx)(g.EditOutlined,{className:"h-4 w-4"}),onClick:()=>eK(!0),children:"Edit Settings"})]}),eV&&eJ?(0,t.jsx)("div",{className:"p-4",children:"Loading..."}):eV?(0,t.jsxs)(M.Form,{form:ek,onFinish:tv,onValuesChange:e=>{if("disable_global_guardrails"in e){let t=!0===e.disable_global_guardrails,l=(ek.getFieldValue("guardrails")||[]).filter(e=>!eX.has(e));ek.setFieldValue("guardrails",t?l:[...Array.from(eX),...l])}},initialValues:{...tw,team_alias:tw.team_alias,models:tw.models,tpm_limit:tw.tpm_limit,rpm_limit:tw.rpm_limit,object_permission_search_tools:tw.object_permission?.search_tools||[],modelLimits:Array.from(new Set([...Object.keys(tw.metadata?.model_tpm_limit??{}),...Object.keys(tw.metadata?.model_rpm_limit??{})])).map(e=>({model:e,tpm:tw.metadata?.model_tpm_limit?.[e],rpm:tw.metadata?.model_rpm_limit?.[e]})),max_budget:tw.max_budget,soft_budget:tw.soft_budget,budget_duration:tw.budget_duration,team_member_tpm_limit:tw.team_member_budget_table?.tpm_limit,team_member_rpm_limit:tw.team_member_budget_table?.rpm_limit,team_member_budget:tw.team_member_budget_table?.max_budget,team_member_budget_duration:tw.team_member_budget_table?.budget_duration,guardrails:tN,policies:tw.policies||[],disable_global_guardrails:tw.metadata?.disable_global_guardrails||!1,soft_budget_alerting_emails:Array.isArray(tw.metadata?.soft_budget_alerting_emails)?tw.metadata.soft_budget_alerting_emails.join(", "):"",metadata:tw.metadata?JSON.stringify((({logging:e,secret_manager_settings:t,soft_budget_alerting_emails:l,model_tpm_limit:a,model_rpm_limit:r,...s})=>s)(tw.metadata),null,2):"",logging_settings:tw.metadata?.logging||[],secret_manager_settings:tw.metadata?.secret_manager_settings?JSON.stringify(tw.metadata.secret_manager_settings,null,2):"",organization_id:tw.organization_id,vector_stores:tw.object_permission?.vector_stores||[],mcp_servers:tw.object_permission?.mcp_servers||[],mcp_access_groups:tw.object_permission?.mcp_access_groups||[],mcp_servers_and_groups:{servers:tw.object_permission?.mcp_servers||[],accessGroups:tw.object_permission?.mcp_access_groups||[],toolsets:tw.object_permission?.mcp_toolsets||[]},mcp_tool_permissions:tw.object_permission?.mcp_tool_permissions||{},agents_and_groups:{agents:tw.object_permission?.agents||[],accessGroups:tw.object_permission?.agent_access_groups||[]},access_group_ids:tw.access_group_ids||[],default_team_member_models:tw.default_team_member_models||[]},layout:"vertical",children:[(0,t.jsx)(M.Form.Item,{label:"Team Name",name:"team_alias",rules:[{required:!0,message:"Please input a team name"}],children:(0,t.jsx)(A.Input,{type:""})}),(0,t.jsx)(M.Form.Item,{label:"Models",name:"models",rules:[{required:!0,message:"Please select at least one model"}],children:(0,t.jsx)(Z.ModelSelect,{value:ek.getFieldValue("models")||[],onChange:e=>ek.setFieldValue("models",e),teamID:e,organizationID:ev?.team_info?.organization_id||void 0,options:{includeSpecialOptions:!0,includeUserModels:!ev?.team_info?.organization_id,showAllProxyModelsOverride:(0,u.isProxyAdminRole)(tn)&&!ev?.team_info?.organization_id},context:"team",dataTestId:"models-select"})}),(0,t.jsx)(M.Form.Item,{label:"Max Budget (USD)",name:"max_budget",children:(0,t.jsx)(el.default,{step:.01,precision:2,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Soft Budget (USD)",name:"soft_budget",children:(0,t.jsx)(el.default,{step:.01,precision:2,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Soft Budget Alerting Emails",name:"soft_budget_alerting_emails",tooltip:"Comma-separated email addresses to receive alerts when the soft budget is reached",children:(0,t.jsx)(A.Input,{placeholder:"example1@test.com, example2@test.com"})}),(0,t.jsxs)(y.Accordion,{className:"mt-4 mb-4",children:[(0,t.jsx)(v.AccordionHeader,{children:(0,t.jsx)("b",{children:"Team Member Settings"})}),(0,t.jsxs)(j.AccordionBody,{children:[(0,t.jsx)(T.Text,{className:"text-xs text-gray-500 mb-4",children:"Optional defaults applied when members join this team. All fields can be overridden per member."}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Default Model Access"," ",(0,t.jsx)(R.Tooltip,{title:"Optional. If set, new members can only access these models by default. Must be a subset of the team's models above. Leave empty to give all members access to all team models.",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),name:"default_team_member_models",children:(0,t.jsx)(M.Form.Item,{noStyle:!0,shouldUpdate:(e,t)=>e.models!==t.models,children:({getFieldValue:e})=>{let l=e("models")||tw.models||[];return(0,t.jsx)(O.Select,{mode:"multiple",placeholder:"Leave empty — all team models accessible to every member",value:ek.getFieldValue("default_team_member_models")||[],onChange:e=>ek.setFieldValue("default_team_member_models",e),options:l.map(e=>({label:e,value:e}))})}})}),(0,t.jsx)(M.Form.Item,{label:"Default Budget (USD)",name:"team_member_budget",tooltip:"Default spend budget for each member in this team.",children:(0,t.jsx)(el.default,{step:.01,precision:2,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Default Budget Duration",name:"team_member_budget_duration",children:(0,t.jsx)(W,{onChange:e=>ek.setFieldValue("team_member_budget_duration",e),value:ek.getFieldValue("team_member_budget_duration")})}),(0,t.jsx)(M.Form.Item,{label:"Default Key Duration (eg: 1d, 1mo)",name:"team_member_key_duration",tooltip:"Set a limit to the duration of a team member's key. Format: 30s (seconds), 30m (minutes), 30h (hours), 30d (days), 1mo (month)",children:(0,t.jsx)(N.TextInput,{placeholder:"e.g., 30d"})}),(0,t.jsx)(M.Form.Item,{label:"Default TPM Limit",name:"team_member_tpm_limit",tooltip:"Default tokens per minute limit for each member. Can be overridden per member.",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"},placeholder:"e.g., 1000"})}),(0,t.jsx)(M.Form.Item,{label:"Default RPM Limit",name:"team_member_rpm_limit",tooltip:"Default requests per minute limit for each member. Can be overridden per member.",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"},placeholder:"e.g., 100"})})]})]}),(0,t.jsx)(M.Form.Item,{label:"Reset Budget",name:"budget_duration",children:(0,t.jsxs)(O.Select,{placeholder:"n/a",children:[(0,t.jsx)(O.Select.Option,{value:"24h",children:"daily"}),(0,t.jsx)(O.Select.Option,{value:"7d",children:"weekly"}),(0,t.jsx)(O.Select.Option,{value:"30d",children:"monthly"})]})}),(0,t.jsx)(M.Form.Item,{label:"Tokens per minute Limit (TPM)",name:"tpm_limit",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Requests per minute Limit (RPM)",name:"rpm_limit",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Model-Specific Rate Limits",tooltip:"Set per-model TPM/RPM limits that apply across the whole team.",children:(0,t.jsx)(M.Form.List,{name:"modelLimits",children:(e,{add:l,remove:a})=>(0,t.jsxs)(t.Fragment,{children:[e.map(({key:e,name:l,...r})=>(0,t.jsxs)(z.Space,{style:{display:"flex",marginBottom:8},align:"baseline",children:[(0,t.jsx)(M.Form.Item,{...r,name:[l,"model"],rules:[{required:!0,message:"Missing model"},{validator:(e,t)=>t&&(ek.getFieldValue("modelLimits")??[]).filter(e=>e?.model===t).length>1?Promise.reject(Error("Duplicate model")):Promise.resolve()}],style:{minWidth:240},children:(0,t.jsx)(O.Select,{showSearch:!0,placeholder:"Select model",allowClear:!0,options:th.map(e=>({value:e,label:e}))})}),(0,t.jsx)(M.Form.Item,{...r,name:[l,"tpm"],rules:[{validator:async(e,t)=>{let a=(ek.getFieldValue("modelLimits")??[])[l]??{};return a.model&&null==t&&null==a.rpm?Promise.reject(Error("Set at least one of TPM or RPM")):Promise.resolve()}}],children:(0,t.jsx)(F.InputNumber,{placeholder:"TPM Limit",min:0})}),(0,t.jsx)(M.Form.Item,{...r,name:[l,"rpm"],children:(0,t.jsx)(F.InputNumber,{placeholder:"RPM Limit",min:0})}),(0,t.jsx)(x.MinusCircleOutlined,{onClick:()=>a(l),style:{color:"#ef4444"}})]},e)),(0,t.jsx)(M.Form.Item,{children:(0,t.jsx)(I.Button,{type:"dashed",onClick:()=>l(),block:!0,icon:(0,t.jsx)(b.PlusOutlined,{}),children:"Add Model Limit"})})]})})}),(0,t.jsx)(M.Form.Item,{label:"Router Settings",children:(0,t.jsx)(ei.default,{ref:tr,accessToken:o||"",value:tw.router_settings?{router_settings:tw.router_settings}:void 0})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Guardrails"," ",(0,t.jsx)(R.Tooltip,{title:"Select which guardrails apply to this team. Global guardrails are enabled by default — uncheck to opt out. Other guardrails are opt-in.",children:(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/guardrails/quick_start",target:"_blank",rel:"noopener noreferrer",onClick:e=>e.stopPropagation(),children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})})]}),name:"guardrails",children:(0,t.jsxs)(O.Select,{mode:"multiple",placeholder:"Select guardrails",optionLabelProp:"label",tagRender:({label:e,value:l,closable:a,onClose:r})=>{let s=eX.has(l);return(0,t.jsxs)(D.Tag,{color:"blue",closable:a,onClose:r,onMouseDown:tk,style:{marginInlineEnd:4},children:[s&&(0,t.jsx)(h.GlobalOutlined,{style:{marginInlineEnd:4},"aria-label":"Global guardrail"}),e]})},children:[(0,t.jsx)(O.Select.OptGroup,{label:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(h.GlobalOutlined,{style:{marginInlineEnd:4}}),"Global"]}),children:(eY?.guardrails??[]).filter(e=>e.litellm_params?.default_on).map(e=>(0,t.jsx)(O.Select.Option,{value:e.guardrail_name,label:e.guardrail_name,disabled:tg,children:e.guardrail_name},e.guardrail_name))}),(0,t.jsx)(O.Select.OptGroup,{label:"Other",children:(eY?.guardrails??[]).filter(e=>!e.litellm_params?.default_on).map(e=>(0,t.jsx)(O.Select.Option,{value:e.guardrail_name,label:e.guardrail_name,children:e.guardrail_name},e.guardrail_name))})]})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Disable all global guardrails"," ",(0,t.jsx)(R.Tooltip,{title:"Kill switch: bypass every global guardrail for this team, including any added in the future. For per-guardrail opt-out instead, use the Guardrails dropdown above.",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),name:"disable_global_guardrails",valuePropName:"checked",children:(0,t.jsx)(P.Switch,{checkedChildren:"Yes",unCheckedChildren:"No"})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Policies"," ",(0,t.jsx)(R.Tooltip,{title:"Apply policies to this team to control guardrails and other settings",children:(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies",target:"_blank",rel:"noopener noreferrer",onClick:e=>e.stopPropagation(),children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})})]}),name:"policies",children:(0,t.jsx)(O.Select,{mode:"tags",placeholder:"Select or enter policies",options:eZ.map(e=>({value:e,label:e}))})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Access Groups"," ",(0,t.jsx)(R.Tooltip,{title:"Assign access groups to this team. Access groups control which models, MCP servers, and agents this team can use",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),name:"access_group_ids",children:(0,t.jsx)(K.default,{placeholder:"Select access groups (optional)"})}),(0,t.jsx)(M.Form.Item,{label:"Vector Stores",name:"vector_stores","aria-label":"Vector Stores",children:(0,t.jsx)(ea.default,{onChange:e=>ek.setFieldValue("vector_stores",e),value:ek.getFieldValue("vector_stores"),accessToken:o||"",placeholder:"Select vector stores"})}),(0,t.jsx)(M.Form.Item,{label:"Allowed Pass Through Routes",name:"allowed_passthrough_routes",children:(0,t.jsx)(q.default,{onChange:e=>ek.setFieldValue("allowed_passthrough_routes",e),value:ek.getFieldValue("allowed_passthrough_routes"),accessToken:o||"",placeholder:"Select pass through routes"})}),(0,t.jsx)(M.Form.Item,{label:"MCP Servers / Access Groups",name:"mcp_servers_and_groups",children:(0,t.jsx)(J.default,{onChange:e=>ek.setFieldValue("mcp_servers_and_groups",e),value:ek.getFieldValue("mcp_servers_and_groups"),accessToken:o||"",placeholder:"Select MCP servers or access groups (optional)"})}),(0,t.jsx)(M.Form.Item,{name:"mcp_tool_permissions",initialValue:{},hidden:!0,children:(0,t.jsx)(A.Input,{type:"hidden"})}),(0,t.jsx)(M.Form.Item,{noStyle:!0,shouldUpdate:(e,t)=>e.mcp_servers_and_groups!==t.mcp_servers_and_groups||e.mcp_tool_permissions!==t.mcp_tool_permissions,children:()=>(0,t.jsx)("div",{className:"mb-6",children:(0,t.jsx)(X.default,{accessToken:o||"",selectedServers:ek.getFieldValue("mcp_servers_and_groups")?.servers||[],toolPermissions:ek.getFieldValue("mcp_tool_permissions")||{},onChange:e=>ek.setFieldsValue({mcp_tool_permissions:e})})})}),(0,t.jsx)(M.Form.Item,{label:"Agents / Access Groups",name:"agents_and_groups",children:(0,t.jsx)($.default,{onChange:e=>ek.setFieldValue("agents_and_groups",e),value:ek.getFieldValue("agents_and_groups"),accessToken:o||"",placeholder:"Select agents or access groups (optional)"})}),(0,t.jsxs)(y.Accordion,{className:"mt-4 mb-4",children:[(0,t.jsx)(v.AccordionHeader,{children:(0,t.jsx)("b",{children:"Search Tool Settings"})}),(0,t.jsx)(j.AccordionBody,{children:(0,t.jsx)(M.Form.Item,{label:"Allowed Search Tools",name:"object_permission_search_tools",tooltip:"Select which search tools this team can access. Leave empty to allow all search tools.",children:(0,t.jsx)(er,{onChange:e=>ek.setFieldValue("object_permission_search_tools",e),value:ek.getFieldValue("object_permission_search_tools"),accessToken:o||"",placeholder:"Select search tools (optional, empty = all allowed)"})})})]}),(0,t.jsx)(M.Form.Item,{label:"Organization",name:"organization_id",children:(0,t.jsx)(O.Select,{allowClear:!0,placeholder:"Select an organization",showSearch:!0,optionFilterProp:"label",options:td.map(e=>({value:e.organization_id,label:e.organization_alias||e.organization_id}))})}),(0,t.jsx)(M.Form.Item,{label:"Logging Settings",name:"logging_settings",children:(0,t.jsx)(es.default,{value:ek.getFieldValue("logging_settings"),onChange:e=>ek.setFieldValue("logging_settings",e)})}),(0,t.jsx)(M.Form.Item,{label:"Secret Manager Settings",name:"secret_manager_settings",help:eg?"Enter secret manager configuration as a JSON object.":"Premium feature - Upgrade to manage secret manager settings.",rules:[{validator:async(e,t)=>{if(!t)return Promise.resolve();try{return JSON.parse(t),Promise.resolve()}catch(e){return Promise.reject(Error("Please enter valid JSON"))}}}],children:(0,t.jsx)(A.Input.TextArea,{rows:6,placeholder:'{"namespace": "admin", "mount": "secret", "path_prefix": "litellm"}',disabled:!eg})}),(0,t.jsx)(M.Form.Item,{label:"Metadata",name:"metadata",children:(0,t.jsx)(A.Input.TextArea,{rows:10})}),(0,t.jsx)("div",{className:"sticky z-10 bg-white p-4 pr-0 border-t border-gray-200 bottom-[-1.5rem] inset-x-[-1.5rem]",children:(0,t.jsxs)("div",{className:"flex justify-end items-center gap-2",children:[(0,t.jsx)(I.Button,{onClick:()=>eK(!1),disabled:tl,children:"Cancel"}),(0,t.jsx)(I.Button,{icon:(0,t.jsx)(_.SaveOutlined,{className:"h-4 w-4"}),type:"primary",htmlType:"submit",loading:tl,children:"Save Changes"})]})})]}):(0,t.jsxs)("div",{className:"space-y-4",children:[(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Team Name"}),(0,t.jsx)("div",{children:tw.team_alias})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Team ID"}),(0,t.jsx)("div",{className:"font-mono",children:tw.team_id})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Created At"}),(0,t.jsx)("div",{children:new Date(tw.created_at).toLocaleString()})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Models"}),(0,t.jsx)("div",{className:"flex flex-wrap gap-2 mt-1",children:tw.models.map((e,l)=>(0,t.jsx)(w.Badge,{color:"red",children:e},l))})]}),tw.default_team_member_models&&tw.default_team_member_models.length>0&&(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Default Member Models"}),(0,t.jsx)("div",{className:"flex flex-wrap gap-2 mt-1",children:tw.default_team_member_models.map((e,l)=>(0,t.jsx)(w.Badge,{color:"blue",children:e},l))})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Rate Limits"}),(0,t.jsxs)("div",{children:["TPM: ",tw.tpm_limit||"Unlimited"]}),(0,t.jsxs)("div",{children:["RPM: ",tw.rpm_limit||"Unlimited"]}),(ef=tw.metadata?.model_tpm_limit??{},ey=tw.metadata?.model_rpm_limit??{},0===(ej=Array.from(new Set([...Object.keys(ef),...Object.keys(ey)]))).length?null:(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsx)(T.Text,{className:"text-gray-500",children:"Per-model limits:"}),ej.map(e=>(0,t.jsxs)("div",{className:"text-xs ml-2",children:[e,": TPM ",ef[e]??"—",", RPM ",ey[e]??"—"]},e))]}))]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Team Budget"}),(0,t.jsxs)("div",{children:["Max Budget:"," ",null!==tw.max_budget?`$${(0,m.formatNumberWithCommas)(tw.max_budget,4)}`:"No Limit"]}),(0,t.jsxs)("div",{children:["Soft Budget:"," ",null!==tw.soft_budget&&void 0!==tw.soft_budget?`$${(0,m.formatNumberWithCommas)(tw.soft_budget,4)}`:"No Limit"]}),(0,t.jsxs)("div",{children:["Budget Reset: ",tw.budget_duration||"Never"]}),tw.metadata?.soft_budget_alerting_emails&&Array.isArray(tw.metadata.soft_budget_alerting_emails)&&tw.metadata.soft_budget_alerting_emails.length>0&&(0,t.jsxs)("div",{children:["Soft Budget Alerting Emails: ",tw.metadata.soft_budget_alerting_emails.join(", ")]})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(T.Text,{className:"font-medium",children:["Team Member Settings"," ",(0,t.jsx)(R.Tooltip,{title:"These are limits on individual team members",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),(0,t.jsxs)("div",{children:["Max Budget: ",tw.team_member_budget_table?.max_budget||"No Limit"]}),(0,t.jsxs)("div",{children:["Budget Duration: ",tw.team_member_budget_table?.budget_duration||"No Limit"]}),(0,t.jsxs)("div",{children:["Key Duration: ",tw.metadata?.team_member_key_duration||"No Limit"]}),(0,t.jsxs)("div",{children:["TPM Limit: ",tw.team_member_budget_table?.tpm_limit||"No Limit"]}),(0,t.jsxs)("div",{children:["RPM Limit: ",tw.team_member_budget_table?.rpm_limit||"No Limit"]})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Router Settings"}),tw.router_settings&&Object.values(tw.router_settings).some(e=>null!=e&&""!==e&&!(Array.isArray(e)&&0===e.length))?(0,t.jsxs)("div",{className:"mt-1 space-y-1",children:[tw.router_settings.routing_strategy&&(0,t.jsxs)("div",{children:["Routing Strategy:"," ",(0,t.jsx)(w.Badge,{color:"blue",children:tw.router_settings.routing_strategy})]}),null!=tw.router_settings.num_retries&&(0,t.jsxs)("div",{children:["Number of Retries: ",tw.router_settings.num_retries]}),null!=tw.router_settings.allowed_fails&&(0,t.jsxs)("div",{children:["Allowed Failures: ",tw.router_settings.allowed_fails]}),null!=tw.router_settings.cooldown_time&&(0,t.jsxs)("div",{children:["Cooldown Time: ",tw.router_settings.cooldown_time,"s"]}),null!=tw.router_settings.timeout&&(0,t.jsxs)("div",{children:["Timeout: ",tw.router_settings.timeout,"s"]}),null!=tw.router_settings.retry_after&&(0,t.jsxs)("div",{children:["Retry After: ",tw.router_settings.retry_after,"s"]}),tw.router_settings.fallbacks&&Array.isArray(tw.router_settings.fallbacks)&&tw.router_settings.fallbacks.length>0&&(0,t.jsxs)("div",{children:["Fallbacks: ",tw.router_settings.fallbacks.length," configured"]}),tw.router_settings.enable_tag_filtering&&(0,t.jsx)("div",{children:"Tag Filtering: Enabled"})]}):(0,t.jsx)("div",{className:"text-gray-400",children:"No router settings configured"})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Organization ID"}),(0,t.jsx)("div",{children:tw.organization_id})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Status"}),(0,t.jsx)(w.Badge,{color:tw.blocked?"red":"green",children:tw.blocked?"Blocked":"Active"})]}),(0,t.jsx)(et.default,{objectPermission:tw.object_permission,variant:"inline",className:"pt-4 border-t border-gray-200",accessToken:o}),(0,t.jsx)(Q,{globalGuardrailNames:eX,teamGuardrails:Array.isArray(tw.metadata?.guardrails)?tw.metadata.guardrails:[],optedOutGlobalGuardrails:Array.isArray(tw.metadata?.opted_out_global_guardrails)?tw.metadata.opted_out_global_guardrails:[],killSwitchOn:tC,variant:"inline",className:"pt-4 border-t border-gray-200"}),(0,t.jsx)(Y.default,{loggingConfigs:tw.metadata?.logging||[],disabledCallbacks:[],variant:"inline",className:"pt-4 border-t border-gray-200"}),tw.metadata?.secret_manager_settings&&(0,t.jsxs)("div",{className:"pt-4 border-t border-gray-200",children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Secret Manager Settings"}),(0,t.jsx)("pre",{className:"mt-2 bg-gray-50 p-3 rounded text-xs overflow-x-auto",children:JSON.stringify(tw.metadata.secret_manager_settings,null,2)})]})]})]})}].filter(e=>tx.includes(e.key))}),(0,t.jsx)(en.default,{visible:eD,onCancel:()=>eR(!1),onSubmit:ty,initialData:eB,mode:"edit",config:{title:"Edit Member",showEmail:!0,showUserId:!0,roleOptions:[{label:"Admin",value:"admin"},{label:"User",value:"user"}],additionalFields:[{name:"max_budget_in_team",label:(0,t.jsxs)("span",{children:["Team Member Budget (USD)"," ",(0,t.jsx)(R.Tooltip,{title:"Maximum amount in USD this member can spend within this team. This is separate from any global user budget limits",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"numerical",step:.01,min:0,placeholder:"Budget limit for this member within this team"},{name:"tpm_limit",label:(0,t.jsxs)("span",{children:["Team Member TPM Limit"," ",(0,t.jsx)(R.Tooltip,{title:"Maximum tokens per minute this member can use within this team. This is separate from any global user TPM limit",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"numerical",step:1,min:0,placeholder:"Tokens per minute limit for this member in this team"},{name:"rpm_limit",label:(0,t.jsxs)("span",{children:["Team Member RPM Limit"," ",(0,t.jsx)(R.Tooltip,{title:"Maximum requests per minute this member can make within this team. This is separate from any global user RPM limit",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"numerical",step:1,min:0,placeholder:"Requests per minute limit for this member in this team"},{name:"allowed_models",label:(0,t.jsxs)("span",{children:["Allowed Models"," ",(0,t.jsx)(R.Tooltip,{title:"Models this member can access within this team. Leave empty to inherit all team models.",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"multi-select",options:(tw.models||[]).map(e=>({label:e,value:e})),placeholder:"Leave empty to inherit all team models"}]}}),(0,t.jsx)(s.default,{isVisible:eT,onCancel:()=>eN(!1),onSubmit:tf,accessToken:o,teamId:e}),(0,t.jsx)(G.default,{isOpen:e9,title:"Delete Team Member",alertMessage:"Removing team members will also delete any keys created by or created for this member.",message:"Are you sure you want to remove this member from the team? This action cannot be undone.",resourceInformationTitle:"Team Member Information",resourceInformation:[{label:"User ID",value:e7?.user_id,code:!0},{label:"Email",value:e7?.user_email},{label:"Role",value:e7?.role}],onCancel:()=>{e8(!1),e6(null)},onOk:tj,confirmLoading:te})]})}],56567)}]); \ No newline at end of file +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,625901,e=>{"use strict";var t=e.i(266027),l=e.i(621482),a=e.i(243652),r=e.i(764205),s=e.i(135214);let i=(0,a.createQueryKeys)("models"),o=(0,a.createQueryKeys)("modelHub"),n=(0,a.createQueryKeys)("allProxyModels");(0,a.createQueryKeys)("selectedTeamModels");let d=(0,a.createQueryKeys)("infiniteModels");e.s(["useAllProxyModels",0,()=>{let{accessToken:e,userId:l,userRole:a}=(0,s.default)();return(0,t.useQuery)({queryKey:n.list({}),queryFn:async()=>await (0,r.modelAvailableCall)(e,l,a,!0,null,!0,!1,"expand"),enabled:!!(e&&l&&a)})},"useInfiniteModelInfo",0,(e=50,t)=>{let{accessToken:a,userId:i,userRole:o}=(0,s.default)();return(0,l.useInfiniteQuery)({queryKey:d.list({filters:{...i&&{userId:i},...o&&{userRole:o},size:e,...t&&{search:t}}}),queryFn:async({pageParam:l})=>await (0,r.modelInfoCall)(a,i,o,l,e,t),initialPageParam:1,getNextPageParam:e=>{if(e.current_page{let{accessToken:e}=(0,s.default)();return(0,t.useQuery)({queryKey:o.list({}),queryFn:async()=>await (0,r.modelHubCall)(e),enabled:!!e})},"useModelsInfo",0,(e=1,l=50,a,o,n,d,m)=>{let{accessToken:c,userId:u,userRole:g}=(0,s.default)();return(0,t.useQuery)({queryKey:i.list({filters:{...u&&{userId:u},...g&&{userRole:g},page:e,size:l,...a&&{search:a},...o&&{modelId:o},...n&&{teamId:n},...d&&{sortBy:d},...m&&{sortOrder:m}}}),queryFn:async()=>await (0,r.modelInfoCall)(c,u,g,e,l,a,o,n,d,m),enabled:!!(c&&u&&g)})}])},91979,e=>{"use strict";e.i(247167);var t=e.i(931067),l=e.i(271645);let a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M909.1 209.3l-56.4 44.1C775.8 155.1 656.2 92 521.9 92 290 92 102.3 279.5 102 511.5 101.7 743.7 289.8 932 521.9 932c181.3 0 335.8-115 394.6-276.1 1.5-4.2-.7-8.9-4.9-10.3l-56.7-19.5a8 8 0 00-10.1 4.8c-1.8 5-3.8 10-5.9 14.9-17.3 41-42.1 77.8-73.7 109.4A344.77 344.77 0 01655.9 829c-42.3 17.9-87.4 27-133.8 27-46.5 0-91.5-9.1-133.8-27A341.5 341.5 0 01279 755.2a342.16 342.16 0 01-73.7-109.4c-17.9-42.4-27-87.4-27-133.9s9.1-91.5 27-133.9c17.3-41 42.1-77.8 73.7-109.4 31.6-31.6 68.4-56.4 109.3-73.8 42.3-17.9 87.4-27 133.8-27 46.5 0 91.5 9.1 133.8 27a341.5 341.5 0 01109.3 73.8c9.9 9.9 19.2 20.4 27.8 31.4l-60.2 47a8 8 0 003 14.1l175.6 43c5 1.2 9.9-2.6 9.9-7.7l.8-180.9c-.1-6.6-7.8-10.3-13-6.2z"}}]},name:"reload",theme:"outlined"};var r=e.i(9583),s=l.forwardRef(function(e,s){return l.createElement(r.default,(0,t.default)({},e,{ref:s,icon:a}))});e.s(["ReloadOutlined",0,s],91979)},969550,e=>{"use strict";var t=e.i(843476),l=e.i(271645);let a=l.forwardRef(function(e,t){return l.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:t},e),l.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M3 4a1 1 0 011-1h16a1 1 0 011 1v2.586a1 1 0 01-.293.707l-6.414 6.414a1 1 0 00-.293.707V17l-4 4v-6.586a1 1 0 00-.293-.707L3.293 7.293A1 1 0 013 6.586V4z"}))});var r=e.i(464571),s=e.i(311451),i=e.i(199133),o=e.i(374009);e.s(["default",0,({options:e,onApplyFilters:n,onResetFilters:d,initialValues:m={},buttonLabel:c="Filters"})=>{let[u,g]=(0,l.useState)(!1),[h,p]=(0,l.useState)(m),[x,b]=(0,l.useState)({}),[_,f]=(0,l.useState)({}),[y,j]=(0,l.useState)({}),[v,w]=(0,l.useState)({}),C=(0,l.useCallback)((0,o.default)(async(e,t)=>{if(t.isSearchable&&t.searchFn){f(e=>({...e,[t.name]:!0}));try{let l=await t.searchFn(e);b(e=>({...e,[t.name]:l}))}catch(e){console.error("Error searching:",e),b(e=>({...e,[t.name]:[]}))}finally{f(e=>({...e,[t.name]:!1}))}}},300),[]),S=(0,l.useCallback)(async e=>{if(e.isSearchable&&e.searchFn&&!v[e.name]){f(t=>({...t,[e.name]:!0})),w(t=>({...t,[e.name]:!0}));try{let t=await e.searchFn("");b(l=>({...l,[e.name]:t}))}catch(t){console.error("Error loading initial options:",t),b(t=>({...t,[e.name]:[]}))}finally{f(t=>({...t,[e.name]:!1}))}}},[v]);(0,l.useEffect)(()=>{u&&e.forEach(e=>{e.isSearchable&&!v[e.name]&&S(e)})},[u,e,S,v]);let T=(e,t)=>{let l={...h,[e]:t};p(l),n(l)};return(0,t.jsxs)("div",{className:"w-full",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 mb-6",children:[(0,t.jsx)(r.Button,{icon:(0,t.jsx)(a,{className:"h-4 w-4"}),onClick:()=>g(!u),className:"flex items-center gap-2",children:c}),(0,t.jsx)(r.Button,{onClick:()=>{let t={};e.forEach(e=>{t[e.name]=""}),p(t),d()},children:"Reset Filters"})]}),u&&(0,t.jsx)("div",{className:"grid grid-cols-3 gap-x-6 gap-y-4 mb-6",children:["Team ID","Status","Organization ID","Key Alias","User ID","End User","Error Code","Error Message","Key Hash","Model","Public model / search tool"].map(l=>{let a,r=e.find(e=>e.label===l||e.name===l);return r?(0,t.jsxs)("div",{className:"flex flex-col gap-2",children:[(0,t.jsx)("label",{className:"text-sm text-gray-600",children:r.label||r.name}),r.isSearchable?(0,t.jsx)(i.Select,{showSearch:!0,className:"w-full",placeholder:`Search ${r.label||r.name}...`,value:h[r.name]||void 0,onChange:e=>T(r.name,e),onOpenChange:e=>{e&&r.isSearchable&&!v[r.name]&&S(r)},onSearch:e=>{j(t=>({...t,[r.name]:e})),r.searchFn&&C(e,r)},filterOption:!1,loading:_[r.name],options:x[r.name]||[],allowClear:!0,notFoundContent:_[r.name]?"Loading...":"No results found"}):r.options?(0,t.jsx)(i.Select,{className:"w-full",placeholder:`Select ${r.label||r.name}...`,value:h[r.name]||void 0,onChange:e=>T(r.name,e),allowClear:!0,children:r.options.map(e=>(0,t.jsx)(i.Select.Option,{value:e.value,children:e.label},e.value))}):r.customComponent?(a=r.customComponent,(0,t.jsx)(a,{value:h[r.name]||void 0,onChange:e=>T(r.name,e??""),placeholder:`Select ${r.label||r.name}...`,allFilters:h})):(0,t.jsx)(s.Input,{className:"w-full",placeholder:`Enter ${r.label||r.name}...`,value:h[r.name]||"",onChange:e=>T(r.name,e.target.value),allowClear:!0})]},r.name):null})})]})}],969550)},633627,e=>{"use strict";var t=e.i(764205);let l=(e,t,l,a)=>{for(let r of e){let e=r?.key_alias;e&&"string"==typeof e&&t.add(e.trim());let s=r?.organization_id??r?.org_id;s&&"string"==typeof s&&l.add(s.trim());let i=r?.user_id;if(i&&"string"==typeof i){let e=r?.user?.user_email||i;a.set(i,e)}}},a=async(e,a)=>{if(!e||!a)return{keyAliases:[],organizationIds:[],userIds:[]};try{let r=new Set,s=new Set,i=new Map,o=await (0,t.keyListCall)(e,null,a,null,null,null,1,100,null,null,"user",null),n=o?.keys||[],d=o?.total_pages??1;l(n,r,s,i);let m=Math.min(d,10)-1;if(m>0){let o=Array.from({length:m},(l,r)=>(0,t.keyListCall)(e,null,a,null,null,null,r+2,100,null,null,"user",null));for(let e of(await Promise.allSettled(o)))"fulfilled"===e.status&&l(e.value?.keys||[],r,s,i)}return{keyAliases:Array.from(r).sort(),organizationIds:Array.from(s).sort(),userIds:Array.from(i.entries()).map(([e,t])=>({id:e,email:t}))}}catch(e){return console.error("Error fetching team filter options:",e),{keyAliases:[],organizationIds:[],userIds:[]}}},r=async(e,l)=>{if(!e)return[];try{let a=[],r=1,s=!0;for(;s;){let i=await (0,t.teamListCall)(e,l||null,null);a=[...a,...i],r{if(!e)return[];try{let l=[],a=1,r=!0;for(;r;){let s=await (0,t.organizationListCall)(e);l=[...l,...s],a{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3"}))});e.s(["ClipboardCopyIcon",0,l],551332)},122577,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M14.752 11.168l-3.197-2.132A1 1 0 0010 9.87v4.263a1 1 0 001.555.832l3.197-2.132a1 1 0 000-1.664z"}),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M21 12a9 9 0 11-18 0 9 9 0 0118 0z"}))});e.s(["PlayIcon",0,l],122577)},902555,e=>{"use strict";var t=e.i(843476),l=e.i(591935),a=e.i(122577),r=e.i(278587),s=e.i(68155),i=e.i(360820),o=e.i(871943),n=e.i(434626),d=e.i(551332),m=e.i(592968),c=e.i(115504),u=e.i(752978);function g({icon:e,onClick:l,className:a,disabled:r,dataTestId:s}){return r?(0,t.jsx)(u.Icon,{icon:e,size:"sm",className:"opacity-50 cursor-not-allowed","data-testid":s}):(0,t.jsx)(u.Icon,{icon:e,size:"sm",onClick:l,className:(0,c.cx)("cursor-pointer",a),"data-testid":s})}let h={Edit:{icon:l.PencilAltIcon,className:"hover:text-blue-600"},Delete:{icon:s.TrashIcon,className:"hover:text-red-600"},Test:{icon:a.PlayIcon,className:"hover:text-blue-600"},Regenerate:{icon:r.RefreshIcon,className:"hover:text-green-600"},Up:{icon:i.ChevronUpIcon,className:"hover:text-blue-600"},Down:{icon:o.ChevronDownIcon,className:"hover:text-blue-600"},Open:{icon:n.ExternalLinkIcon,className:"hover:text-green-600"},Copy:{icon:d.ClipboardCopyIcon,className:"hover:text-blue-600"}};function p({onClick:e,tooltipText:l,disabled:a=!1,disabledTooltipText:r,dataTestId:s,variant:i}){let{icon:o,className:n}=h[i];return(0,t.jsx)(m.Tooltip,{title:a?r:l,children:(0,t.jsx)("span",{children:(0,t.jsx)(g,{icon:o,onClick:e,className:n,disabled:a,dataTestId:s})})})}e.s(["default",()=>p],902555)},434626,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14"}))});e.s(["ExternalLinkIcon",0,l],434626)},278587,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"}))});e.s(["RefreshIcon",0,l],278587)},207670,e=>{"use strict";function t(){for(var e,t,l=0,a="",r=arguments.length;lt,"default",0,t])},728889,e=>{"use strict";var t=e.i(290571),l=e.i(271645),a=e.i(829087),r=e.i(480731),s=e.i(444755),i=e.i(673706),o=e.i(95779);let n={xs:{paddingX:"px-1.5",paddingY:"py-1.5"},sm:{paddingX:"px-1.5",paddingY:"py-1.5"},md:{paddingX:"px-2",paddingY:"py-2"},lg:{paddingX:"px-2",paddingY:"py-2"},xl:{paddingX:"px-2.5",paddingY:"py-2.5"}},d={xs:{height:"h-3",width:"w-3"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-7",width:"w-7"},xl:{height:"h-9",width:"w-9"}},m={simple:{rounded:"",border:"",ring:"",shadow:""},light:{rounded:"rounded-tremor-default",border:"",ring:"",shadow:""},shadow:{rounded:"rounded-tremor-default",border:"border",ring:"",shadow:"shadow-tremor-card dark:shadow-dark-tremor-card"},solid:{rounded:"rounded-tremor-default",border:"border-2",ring:"ring-1",shadow:""},outlined:{rounded:"rounded-tremor-default",border:"border",ring:"ring-2",shadow:""}},c=(0,i.makeClassName)("Icon"),u=l.default.forwardRef((e,u)=>{let{icon:g,variant:h="simple",tooltip:p,size:x=r.Sizes.SM,color:b,className:_}=e,f=(0,t.__rest)(e,["icon","variant","tooltip","size","color","className"]),y=((e,t)=>{switch(e){case"simple":return{textColor:t?(0,i.getColorClassNames)(t,o.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:"",borderColor:"",ringColor:""};case"light":return{textColor:t?(0,i.getColorClassNames)(t,o.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,o.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand-muted dark:bg-dark-tremor-brand-muted",borderColor:"",ringColor:""};case"shadow":return{textColor:t?(0,i.getColorClassNames)(t,o.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,o.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:"border-tremor-border dark:border-dark-tremor-border",ringColor:""};case"solid":return{textColor:t?(0,i.getColorClassNames)(t,o.colorPalette.text).textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,o.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand dark:bg-dark-tremor-brand",borderColor:"border-tremor-brand-inverted dark:border-dark-tremor-brand-inverted",ringColor:"ring-tremor-ring dark:ring-dark-tremor-ring"};case"outlined":return{textColor:t?(0,i.getColorClassNames)(t,o.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,o.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:t?(0,i.getColorClassNames)(t,o.colorPalette.ring).borderColor:"border-tremor-brand-subtle dark:border-dark-tremor-brand-subtle",ringColor:t?(0,s.tremorTwMerge)((0,i.getColorClassNames)(t,o.colorPalette.ring).ringColor,"ring-opacity-40"):"ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted"}}})(h,b),{tooltipProps:j,getReferenceProps:v}=(0,a.useTooltip)();return l.default.createElement("span",Object.assign({ref:(0,i.mergeRefs)([u,j.refs.setReference]),className:(0,s.tremorTwMerge)(c("root"),"inline-flex shrink-0 items-center justify-center",y.bgColor,y.textColor,y.borderColor,y.ringColor,m[h].rounded,m[h].border,m[h].shadow,m[h].ring,n[x].paddingX,n[x].paddingY,_)},v,f),l.default.createElement(a.default,Object.assign({text:p},j)),l.default.createElement(g,{className:(0,s.tremorTwMerge)(c("icon"),"shrink-0",d[x].height,d[x].width)}))});u.displayName="Icon",e.s(["default",()=>u],728889)},752978,e=>{"use strict";var t=e.i(728889);e.s(["Icon",()=>t.default])},591935,e=>{"use strict";var t=e.i(271645);let l=t.forwardRef(function(e,l){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:l},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"}))});e.s(["PencilAltIcon",0,l],591935)},907308,e=>{"use strict";var t=e.i(843476),l=e.i(271645),a=e.i(212931),r=e.i(808613),s=e.i(464571),i=e.i(199133),o=e.i(592968),n=e.i(213205),d=e.i(374009),m=e.i(764205);e.s(["default",0,({isVisible:e,onCancel:c,onSubmit:u,accessToken:g,title:h="Add Team Member",roles:p=[{label:"admin",value:"admin",description:"Admin role. Can create team keys, add members, and manage settings."},{label:"user",value:"user",description:"User role. Can view team info, but not manage it."}],defaultRole:x="user",teamId:b})=>{let[_]=r.Form.useForm(),[f,y]=(0,l.useState)([]),[j,v]=(0,l.useState)(!1),[w,C]=(0,l.useState)("user_email"),[S,T]=(0,l.useState)(!1),N=async(e,t)=>{if(!e)return void y([]);v(!0);try{let l=new URLSearchParams;if(l.append(t,e),b&&l.append("team_id",b),null==g)return;let a=(await (0,m.userFilterUICall)(g,l)).map(e=>({label:"user_email"===t?`${e.user_email}`:`${e.user_id}`,value:"user_email"===t?e.user_email:e.user_id,user:e}));y(a)}catch(e){console.error("Error fetching users:",e)}finally{v(!1)}},k=(0,l.useCallback)((0,d.default)((e,t)=>N(e,t),300),[]),I=(e,t)=>{C(t),k(e,t)},M=(e,t)=>{let l=t.user;_.setFieldsValue({user_email:l.user_email,user_id:l.user_id,role:_.getFieldValue("role")})},A=async e=>{T(!0);try{await u(e)}finally{T(!1)}};return(0,t.jsx)(a.Modal,{title:h,open:e,onCancel:()=>{_.resetFields(),y([]),c()},footer:null,width:800,maskClosable:!S,children:(0,t.jsxs)(r.Form,{form:_,onFinish:A,labelCol:{span:8},wrapperCol:{span:16},labelAlign:"left",initialValues:{role:x},children:[(0,t.jsx)(r.Form.Item,{label:"Email",name:"user_email",className:"mb-4",children:(0,t.jsx)(i.Select,{showSearch:!0,className:"w-full",placeholder:"Search by email",filterOption:!1,onSearch:e=>I(e,"user_email"),onSelect:(e,t)=>M(e,t),options:"user_email"===w?f:[],loading:j,allowClear:!0,"data-testid":"member-email-search"})}),(0,t.jsx)("div",{className:"text-center mb-4",children:"OR"}),(0,t.jsx)(r.Form.Item,{label:"User ID",name:"user_id",className:"mb-4",children:(0,t.jsx)(i.Select,{showSearch:!0,className:"w-full",placeholder:"Search by user ID",filterOption:!1,onSearch:e=>I(e,"user_id"),onSelect:(e,t)=>M(e,t),options:"user_id"===w?f:[],loading:j,allowClear:!0})}),(0,t.jsx)(r.Form.Item,{label:"Member Role",name:"role",className:"mb-4",children:(0,t.jsx)(i.Select,{defaultValue:x,children:p.map(e=>(0,t.jsx)(i.Select.Option,{value:e.value,children:(0,t.jsxs)(o.Tooltip,{title:e.description,children:[(0,t.jsx)("span",{className:"font-medium",children:e.label}),(0,t.jsxs)("span",{className:"ml-2 text-gray-500 text-sm",children:["- ",e.description]})]})},e.value))})}),(0,t.jsx)("div",{className:"text-right mt-4",children:(0,t.jsx)(s.Button,{type:"primary",htmlType:"submit",icon:(0,t.jsx)(n.UserAddOutlined,{}),loading:S,children:S?"Adding...":"Add Member"})})]})})}])},162386,e=>{"use strict";var t=e.i(843476),l=e.i(625901),a=e.i(109799),r=e.i(785242),s=e.i(738014),i=e.i(199133),o=e.i(981339),n=e.i(592968);let d={label:"All Proxy Models",value:"all-proxy-models"},m={label:"No Default Models",value:"no-default-models"},c=[d,m],u={user:({allProxyModels:e,userModels:t,options:l})=>t&&l?.includeUserModels?t:[],team:({allProxyModels:e,selectedOrganization:t,userModels:l})=>t?t.models.includes(d.value)||0===t.models.length?e:e.filter(e=>t.models.includes(e)):e??[],organization:({allProxyModels:e})=>e,global:({allProxyModels:e})=>e};e.s(["ModelSelect",0,e=>{let{teamID:g,organizationID:h,options:p,context:x,dataTestId:b,value:_=[],onChange:f,style:y}=e,{includeUserModels:j,showAllTeamModelsOption:v,showAllProxyModelsOverride:w,includeSpecialOptions:C}=p||{},{data:S,isLoading:T}=(0,l.useAllProxyModels)(),{data:N,isLoading:k}=(0,r.useTeam)(g),{data:I,isLoading:M}=(0,a.useOrganization)(h),{data:A,isLoading:F}=(0,s.useCurrentUser)(),O=e=>c.some(t=>t.value===e),z=_.some(O),P=I?.models.includes(d.value)||I?.models.length===0;if(T||k||M||F)return(0,t.jsx)(o.Skeleton.Input,{active:!0,block:!0});let{wildcard:L,regular:D}=(e=>{let t=[],l=[];for(let a of e)a.endsWith("/*")?t.push(a):l.push(a);return{wildcard:t,regular:l}})(((e,t,l)=>{let a=Array.from(new Map(e.map(e=>[e.id,e])).values()).map(e=>e.id);if(t.options?.showAllProxyModelsOverride)return a;let r=u[t.context];return r?r({allProxyModels:a,...l,options:t.options}):[]})(S?.data??[],e,{selectedTeam:N,selectedOrganization:I,userModels:A?.models}));return(0,t.jsx)(i.Select,{"data-testid":b,value:_,onChange:e=>{let t=e.filter(O);f(t.length>0?[t[t.length-1]]:e)},style:y,options:[...C?[{label:(0,t.jsx)("span",{children:"Special Options"}),title:"Special Options",options:[...w||P&&C||"global"===x?[{label:(0,t.jsx)("span",{children:"All Proxy Models"}),value:d.value,disabled:_.length>0&&_.some(e=>O(e)&&e!==d.value),key:d.value}]:[],{label:(0,t.jsx)("span",{children:"No Default Models"}),value:m.value,disabled:_.length>0&&_.some(e=>O(e)&&e!==m.value),key:m.value}]}]:[],...L.length>0?[{label:(0,t.jsx)("span",{children:"Wildcard Options"}),title:"Wildcard Options",options:L.map(e=>{let l=e.replace("/*",""),a=l.charAt(0).toUpperCase()+l.slice(1);return{label:(0,t.jsx)("span",{children:`All ${a} models`}),value:e,disabled:z}})}]:[],{label:(0,t.jsx)("span",{children:"Models"}),title:"Models",options:D.map(e=>({label:(0,t.jsx)("span",{children:e}),value:e,disabled:z}))}],mode:"multiple",placeholder:"Select Models",allowClear:!0,maxTagCount:"responsive",maxTagPlaceholder:e=>(0,t.jsx)(n.Tooltip,{styles:{root:{pointerEvents:"none"}},title:e.map(({value:e})=>e).join(", "),children:(0,t.jsxs)("span",{children:["+",e.length," more"]})})})}],162386)},276173,e=>{"use strict";var t=e.i(843476),l=e.i(599724),a=e.i(779241),r=e.i(464571),s=e.i(808613),i=e.i(212931),o=e.i(199133),n=e.i(271645),d=e.i(435451);e.s(["default",0,({visible:e,onCancel:m,onSubmit:c,initialData:u,mode:g,config:h})=>{let p,[x]=s.Form.useForm(),[b,_]=(0,n.useState)(!1);console.log("Initial Data:",u),(0,n.useEffect)(()=>{if(e)if("edit"===g&&u){let e={...u,role:u.role||h.defaultRole,max_budget_in_team:u.max_budget_in_team||null,tpm_limit:u.tpm_limit||null,rpm_limit:u.rpm_limit||null,allowed_models:u.allowed_models||[]};console.log("Setting form values:",e),x.setFieldsValue(e)}else x.resetFields(),x.setFieldsValue({role:h.defaultRole||h.roleOptions[0]?.value})},[e,u,g,x,h.defaultRole,h.roleOptions]);let f=async e=>{try{_(!0);let t=Object.entries(e).reduce((e,[t,l])=>{if("string"==typeof l){let a=l.trim();return""===a&&("max_budget_in_team"===t||"tpm_limit"===t||"rpm_limit"===t)?{...e,[t]:null}:{...e,[t]:a}}return{...e,[t]:l}},{});console.log("Submitting form data:",t),await Promise.resolve(c(t)),x.resetFields()}catch(e){console.error("Form submission error:",e)}finally{_(!1)}};return(0,t.jsx)(i.Modal,{title:h.title||("add"===g?"Add Member":"Edit Member"),open:e,width:1e3,footer:null,onCancel:m,children:(0,t.jsxs)(s.Form,{form:x,onFinish:f,labelCol:{span:8},wrapperCol:{span:16},labelAlign:"left",children:[h.showEmail&&(0,t.jsx)(s.Form.Item,{label:"Email",name:"user_email",className:"mb-4",rules:[{type:"email",message:"Please enter a valid email!"}],children:(0,t.jsx)(a.TextInput,{placeholder:"user@example.com"})}),h.showEmail&&h.showUserId&&(0,t.jsx)("div",{className:"text-center mb-4",children:(0,t.jsx)(l.Text,{children:"OR"})}),h.showUserId&&(0,t.jsx)(s.Form.Item,{label:"User ID",name:"user_id",className:"mb-4",children:(0,t.jsx)(a.TextInput,{placeholder:"user_123"})}),(0,t.jsx)(s.Form.Item,{label:(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)("span",{children:"Role"}),"edit"===g&&u&&(0,t.jsxs)("span",{className:"text-gray-500 text-sm",children:["(Current: ",(p=u.role,h.roleOptions.find(e=>e.value===p)?.label||p),")"]})]}),name:"role",className:"mb-4",rules:[{required:!0,message:"Please select a role!"}],children:(0,t.jsx)(o.Select,{children:"edit"===g&&u?[...h.roleOptions.filter(e=>e.value===u.role),...h.roleOptions.filter(e=>e.value!==u.role)].map(e=>(0,t.jsx)(o.Select.Option,{value:e.value,children:e.label},e.value)):h.roleOptions.map(e=>(0,t.jsx)(o.Select.Option,{value:e.value,children:e.label},e.value))})}),h.additionalFields?.map(e=>(0,t.jsx)(s.Form.Item,{label:e.label,name:e.name,className:"mb-4",rules:e.rules,children:(e=>{switch(e.type){case"input":return(0,t.jsx)(a.TextInput,{placeholder:e.placeholder});case"numerical":return(0,t.jsx)(d.default,{step:e.step||1,min:e.min||0,style:{width:"100%"},placeholder:e.placeholder||"Enter a numerical value"});case"select":return(0,t.jsx)(o.Select,{children:e.options?.map(e=>(0,t.jsx)(o.Select.Option,{value:e.value,children:e.label},e.value))});case"multi-select":return(0,t.jsx)(o.Select,{mode:"multiple",placeholder:e.placeholder||"Select options",options:e.options,allowClear:!0});default:return null}})(e)},e.name)),(0,t.jsxs)("div",{className:"text-right mt-6",children:[(0,t.jsx)(r.Button,{onClick:m,className:"mr-2",disabled:b,children:"Cancel"}),(0,t.jsx)(r.Button,{type:"default",htmlType:"submit",loading:b,children:"add"===g?b?"Adding...":"Add Member":b?"Saving...":"Save Changes"})]})]})})}])},294612,e=>{"use strict";var t=e.i(843476),l=e.i(100486),a=e.i(827252),r=e.i(213205),s=e.i(771674),i=e.i(464571),o=e.i(770914),n=e.i(291542),d=e.i(262218),m=e.i(592968),c=e.i(898586),u=e.i(902555);let{Text:g}=c.Typography;function h({members:e,canEdit:c,onEdit:h,onDelete:p,onAddMember:x,roleColumnTitle:b="Role",roleTooltip:_,extraColumns:f=[],showDeleteForMember:y,emptyText:j}){let v=[{title:"User Email",dataIndex:"user_email",key:"user_email",render:e=>(0,t.jsx)(g,{children:e||"-"})},{title:"User ID",dataIndex:"user_id",key:"user_id",render:e=>"default_user_id"===e?(0,t.jsx)(d.Tag,{color:"blue",children:"Default Proxy Admin"}):(0,t.jsx)(g,{children:e||"-"})},{title:_?(0,t.jsxs)(o.Space,{direction:"horizontal",children:[b,(0,t.jsx)(m.Tooltip,{title:_,children:(0,t.jsx)(a.InfoCircleOutlined,{})})]}):b,dataIndex:"role",key:"role",render:e=>(0,t.jsxs)(o.Space,{children:[e?.toLowerCase()==="admin"||e?.toLowerCase()==="org_admin"?(0,t.jsx)(l.CrownOutlined,{}):(0,t.jsx)(s.UserOutlined,{}),(0,t.jsx)(g,{style:{textTransform:"capitalize"},children:e||"-"})]})},...f,{title:"Actions",key:"actions",fixed:"right",width:120,render:(e,l)=>c?(0,t.jsxs)(o.Space,{children:[(0,t.jsx)(u.default,{variant:"Edit",tooltipText:"Edit member",dataTestId:"edit-member",onClick:()=>h(l)}),(!y||y(l))&&(0,t.jsx)(u.default,{variant:"Delete",tooltipText:"Delete member",dataTestId:"delete-member",onClick:()=>p(l)})]}):null}];return(0,t.jsxs)(o.Space,{direction:"vertical",style:{width:"100%"},children:[(0,t.jsxs)("span",{className:"inline-flex text-sm text-gray-700",children:[e.length," Member",1!==e.length?"s":""]}),(0,t.jsx)(n.Table,{columns:v,dataSource:e,rowKey:e=>e.user_id??e.user_email??JSON.stringify(e),pagination:!1,size:"small",scroll:{x:"max-content"},locale:j?{emptyText:j}:void 0}),x&&c&&(0,t.jsx)(i.Button,{icon:(0,t.jsx)(r.UserAddOutlined,{}),type:"primary",onClick:x,children:"Add Member"})]})}e.s(["default",()=>h])},56567,838932,471145,e=>{"use strict";var t=e.i(843476),l=e.i(135214),a=e.i(109799),r=e.i(912598),s=e.i(907308),i=e.i(764205),o=e.i(266027);let n=(0,e.i(243652).createQueryKeys)("guardrails"),d=()=>{let{accessToken:e,userId:t,userRole:a}=(0,l.default)();return(0,o.useQuery)({queryKey:n.list({}),queryFn:async()=>(0,i.getGuardrailsList)(e),enabled:!!(e&&t&&a),select:e=>{let t=e?.guardrails??[],l=new Set,a=new Set;for(let e of t)e.litellm_params?.default_on?l.add(e.guardrail_name):a.add(e.guardrail_name);return{guardrails:t,globalGuardrailNames:l,optionalGuardrailNames:a}}})};e.s(["useGuardrails",0,d],838932);var m=e.i(500330),c=e.i(11751),u=e.i(708347),g=e.i(751904),h=e.i(160818),p=e.i(827252),x=e.i(564897),b=e.i(646563),_=e.i(987432),f=e.i(530212),y=e.i(677667),j=e.i(130643),v=e.i(898667),w=e.i(389083),C=e.i(304967),S=e.i(350967),T=e.i(599724),N=e.i(779241),k=e.i(629569),I=e.i(464571),M=e.i(808613),A=e.i(311451),F=e.i(28651),O=e.i(199133),z=e.i(770914),P=e.i(790848),L=e.i(653496),D=e.i(262218),R=e.i(592968),E=e.i(888259),B=e.i(678784),U=e.i(118366),V=e.i(271645),K=e.i(9314),$=e.i(552130),G=e.i(127952);function W({className:e,value:l,onChange:a}){return(0,t.jsxs)(O.Select,{className:e,value:l,onChange:a,children:[(0,t.jsx)(O.Select.Option,{value:"24h",children:"Daily"}),(0,t.jsx)(O.Select.Option,{value:"7d",children:"Weekly"}),(0,t.jsx)(O.Select.Option,{value:"30d",children:"Monthly"})]})}var q=e.i(844565),H=e.i(355619);let Q=function({globalGuardrailNames:e,teamGuardrails:l=[],optedOutGlobalGuardrails:a=[],killSwitchOn:r=!1,variant:s="card",className:i=""}){let o=new Set(a),n=Array.from(e).filter(e=>!o.has(e)),d=l.filter(t=>!e.has(t)),m=r||0!==n.length||0!==d.length?(0,t.jsxs)("div",{className:"flex flex-col gap-4",children:[(0,t.jsxs)("div",{children:[(0,t.jsxs)("span",{className:"block text-sm font-medium text-gray-700 mb-2",children:[(0,t.jsx)(h.GlobalOutlined,{style:{marginInlineEnd:4},"aria-label":"Global guardrail"}),"Global"]}),r?(0,t.jsx)(D.Tag,{color:"gold",children:"Bypassed for this team"}):n.length>0?(0,t.jsx)("div",{className:"flex flex-wrap gap-2",children:n.map(e=>(0,t.jsx)(D.Tag,{color:"blue",children:e},e))}):(0,t.jsx)("span",{className:"block text-sm text-gray-500",children:"None configured"})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)("span",{className:"block text-sm font-medium text-gray-700 mb-2",children:"Team-specific"}),d.length>0?(0,t.jsx)("div",{className:"flex flex-wrap gap-2",children:d.map(e=>(0,t.jsx)(D.Tag,{color:"blue",children:e},e))}):(0,t.jsx)("span",{className:"block text-sm text-gray-500",children:"None configured"})]})]}):(0,t.jsx)("span",{className:"block text-gray-500",children:"No guardrails configured"});return"card"===s?(0,t.jsxs)("div",{className:`bg-white border border-gray-200 rounded-lg p-6 ${i}`,children:[(0,t.jsx)("div",{className:"flex items-center gap-2 mb-6",children:(0,t.jsxs)("div",{children:[(0,t.jsx)("span",{className:"block font-semibold text-gray-900",children:"Guardrails Settings"}),(0,t.jsx)("span",{className:"block text-xs text-gray-500",children:"Global and team-specific guardrails applied to this team"})]})}),m]}):(0,t.jsxs)("div",{className:`${i}`,children:[(0,t.jsx)("span",{className:"block font-medium text-gray-900 mb-3",children:"Guardrails Settings"}),m]})};var Y=e.i(643449),J=e.i(75921),X=e.i(390605),Z=e.i(162386),ee=e.i(727749),et=e.i(384767),el=e.i(435451),ea=e.i(916940);let er=({onChange:e,value:l,className:a,accessToken:r,placeholder:s="Select search tools (optional)",disabled:o=!1})=>{let[n,d]=(0,V.useState)([]),[m,c]=(0,V.useState)(!1);return(0,V.useEffect)(()=>{(async()=>{if(r){c(!0);try{let e=await (0,i.fetchSearchTools)(r),t=Array.isArray(e?.search_tools)?e.search_tools:Array.isArray(e?.data)?e.data:[];d(t.map(e=>e?.search_tool_name).filter(e=>"string"==typeof e&&e.length>0).map(e=>({label:e,value:e})))}catch(e){console.error("Failed to load search tools:",e)}finally{c(!1)}}})()},[r]),(0,t.jsx)(O.Select,{mode:"multiple",allowClear:!0,showSearch:!0,optionFilterProp:"label",placeholder:s,onChange:e,value:l,loading:m,className:a,options:n,style:{width:"100%"},disabled:o})};e.s(["default",0,er],471145);var es=e.i(183588),ei=e.i(460285),eo=e.i(276173),en=e.i(91979),ed=e.i(269200),em=e.i(942232),ec=e.i(977572),eu=e.i(427612),eg=e.i(64848),eh=e.i(496020),ep=e.i(536916),ex=e.i(21548);let eb={"/key/generate":"Member can generate a virtual key for this team","/key/service-account/generate":"Member can generate a service account key (not belonging to any user) for this team","/key/update":"Member can update a virtual key belonging to this team","/key/delete":"Member can delete a virtual key belonging to this team","/key/info":"Member can get info about a virtual key belonging to this team","/key/regenerate":"Member can regenerate a virtual key belonging to this team","/key/{key_id}/regenerate":"Member can regenerate a virtual key belonging to this team","/key/list":"Member can list virtual keys belonging to this team","/key/block":"Member can block a virtual key belonging to this team","/key/unblock":"Member can unblock a virtual key belonging to this team","/team/daily/activity":"Member can view all team usage data (not just their own)","/spend/logs":"Member can view spend logs for the entire team (not just their own)"},e_=({teamId:e,accessToken:l,canEditTeam:a})=>{let[r,s]=(0,V.useState)([]),[o,n]=(0,V.useState)([]),[d,m]=(0,V.useState)(!0),[c,u]=(0,V.useState)(!1),[g,h]=(0,V.useState)(!1),p=async()=>{try{if(m(!0),!l)return;let t=await (0,i.getTeamPermissionsCall)(l,e),a=t.all_available_permissions||[];s(a);let r=t.team_member_permissions||[];n(r),h(!1)}catch(e){ee.default.fromBackend("Failed to load permissions"),console.error("Error fetching permissions:",e)}finally{m(!1)}};(0,V.useEffect)(()=>{p()},[e,l]);let x=async()=>{try{if(!l)return;u(!0),await (0,i.teamPermissionsUpdateCall)(l,e,o),ee.default.success("Permissions updated successfully"),h(!1)}catch(e){ee.default.fromBackend("Failed to update permissions"),console.error("Error updating permissions:",e)}finally{u(!1)}};if(d)return(0,t.jsx)("div",{className:"p-6 text-center",children:"Loading permissions..."});let b=r.length>0;return(0,t.jsxs)(C.Card,{className:"bg-white shadow-md rounded-md p-6",children:[(0,t.jsxs)("div",{className:"flex flex-col sm:flex-row justify-between items-start sm:items-center border-b pb-4 mb-6",children:[(0,t.jsx)(k.Title,{className:"mb-2 sm:mb-0",children:"Member Permissions"}),a&&g&&(0,t.jsxs)("div",{className:"flex gap-3",children:[(0,t.jsx)(I.Button,{icon:(0,t.jsx)(en.ReloadOutlined,{}),onClick:()=>{p()},children:"Reset"}),(0,t.jsx)(I.Button,{onClick:x,loading:c,type:"primary",icon:(0,t.jsx)(_.SaveOutlined,{}),children:"Save Changes"})]})]}),(0,t.jsx)(T.Text,{className:"mb-6 text-gray-600",children:"Control what team members can do when they are not team admins."}),b?(0,t.jsx)("div",{className:"overflow-x-auto",children:(0,t.jsxs)(ed.Table,{className:" min-w-full",children:[(0,t.jsx)(eu.TableHead,{children:(0,t.jsxs)(eh.TableRow,{children:[(0,t.jsx)(eg.TableHeaderCell,{children:"Method"}),(0,t.jsx)(eg.TableHeaderCell,{children:"Endpoint"}),(0,t.jsx)(eg.TableHeaderCell,{children:"Description"}),(0,t.jsx)(eg.TableHeaderCell,{className:"sticky right-0 bg-white shadow-[-4px_0_4px_-4px_rgba(0,0,0,0.1)] text-center",children:"Allow Access"})]})}),(0,t.jsx)(em.TableBody,{children:r.map(e=>{let l=(e=>{let t=e.includes("/info")||e.includes("/list")||e.includes("/activity")||"/spend/logs"===e?"GET":"POST",l=eb[e];if(!l){for(let[t,a]of Object.entries(eb))if(e.includes(t)){l=a;break}}return l||(l=`Access ${e}`),{method:t,endpoint:e,description:l,route:e}})(e);return(0,t.jsxs)(eh.TableRow,{className:"hover:bg-gray-50 transition-colors",children:[(0,t.jsx)(ec.TableCell,{children:(0,t.jsx)("span",{className:`px-2 py-1 rounded text-xs font-medium ${"GET"===l.method?"bg-blue-100 text-blue-800":"bg-green-100 text-green-800"}`,children:l.method})}),(0,t.jsx)(ec.TableCell,{children:(0,t.jsx)("span",{className:"font-mono text-sm text-gray-800",children:l.endpoint})}),(0,t.jsx)(ec.TableCell,{className:"text-gray-700",children:l.description}),(0,t.jsx)(ec.TableCell,{className:"sticky right-0 bg-white shadow-[-4px_0_4px_-4px_rgba(0,0,0,0.1)] text-center",children:(0,t.jsx)(ep.Checkbox,{checked:o.includes(e),onChange:t=>{n(t.target.checked?[...o,e]:o.filter(t=>t!==e)),h(!0)},disabled:!a})})]},e)})})]})}):(0,t.jsx)("div",{className:"py-12",children:(0,t.jsx)(ex.Empty,{description:"No permissions available"})})]})};var ef=e.i(822315);function ey(e){if(!e)return null;let t=(0,ef.default)(e);return t.isValid()?t.format("MMM D, YYYY"):null}var ej=e.i(175712),ev=e.i(178654),ew=e.i(621192),eC=e.i(898586);let eS=async(e,t)=>{let l=(0,i.getProxyBaseUrl)(),a=l?`${l}/team/${encodeURIComponent(t)}/members/me`:`/team/${encodeURIComponent(t)}/members/me`,r=await fetch(a,{method:"GET",headers:{[(0,i.getGlobalLitellmHeaderName)()]:`Bearer ${e}`,"Content-Type":"application/json"}});if(404===r.status)return null;if(!r.ok){let e=await r.json().catch(()=>({}));throw Error((0,i.deriveErrorMessage)(e))}return await r.json()},eT=(e,l)=>(0,t.jsxs)(z.Space,{size:4,children:[(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:e}),(0,t.jsx)(R.Tooltip,{title:l,children:(0,t.jsx)(p.InfoCircleOutlined,{style:{color:"#8c8c8c"}})})]}),eN=(e,t=4)=>null==e?"0":(0,m.formatNumberWithCommas)(e,t),ek=e=>null==e?"Unlimited":(0,m.formatNumberWithCommas)(e,0);function eI({teamId:e}){let{data:a,isLoading:r,error:s}=(e=>{let{accessToken:t}=(0,l.default)();return(0,o.useQuery)({queryKey:["team",e,"members","me"],queryFn:()=>eS(t,e),enabled:!!(t&&e)})})(e);if(r)return(0,t.jsx)(ej.Card,{children:(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"Loading your membership info…"})});if(s)return(0,t.jsx)(ej.Card,{children:(0,t.jsx)(eC.Typography.Text,{type:"danger",children:s instanceof Error?s.message:"Failed to load your membership info for this team."})});if(!a)return(0,t.jsx)(ej.Card,{children:(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"No membership info available for the current user in this team."})});let i=a.litellm_budget_table??null,n=i?.max_budget??null,d=a.spend??0,m=a.total_spend??0,c=i?.tpm_limit??null,u=i?.rpm_limit??null,g=ey(i?.budget_reset_at),h=i?.allowed_models??null;return(0,t.jsxs)(z.Space,{direction:"vertical",size:"middle",style:{width:"100%"},children:[(0,t.jsx)(ej.Card,{children:(0,t.jsxs)(ew.Row,{gutter:[24,16],children:[(0,t.jsxs)(ev.Col,{xs:24,sm:12,md:8,children:[(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"User"}),(0,t.jsx)("div",{style:{marginTop:4},children:(0,t.jsx)(eC.Typography.Text,{strong:!0,children:a.user_email||a.user_id})}),(0,t.jsx)(eC.Typography.Text,{type:"secondary",style:{fontSize:12,fontFamily:"monospace"},children:a.user_id})]}),(0,t.jsxs)(ev.Col,{xs:24,sm:12,md:8,children:[(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"Team Role"}),(0,t.jsx)("div",{style:{marginTop:4},children:(0,t.jsx)(D.Tag,{color:"admin"===a.role?"blue":"default",children:a.role||"user"})})]})]})}),(0,t.jsxs)(ew.Row,{gutter:[16,16],children:[(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Current Cycle Spend (USD)","Spend for the current budget cycle. Resets to $0 when the budget window rolls over."),(0,t.jsxs)("div",{style:{marginTop:8},children:[(0,t.jsxs)(eC.Typography.Title,{level:3,style:{margin:0},children:["$",eN(d,4)]}),(0,t.jsxs)(eC.Typography.Text,{type:"secondary",children:["of ",null===n?"Unlimited":`$${eN(n,4)}`]})]}),g&&(0,t.jsx)("div",{style:{marginTop:4},children:(0,t.jsxs)(eC.Typography.Text,{type:"secondary",children:["Resets ",g]})})]})}),(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Rate Limits","Your per-member rate limits within this team."),(0,t.jsxs)("div",{style:{marginTop:8},children:[(0,t.jsxs)(eC.Typography.Text,{children:["TPM: ",ek(c)]}),(0,t.jsx)("br",{}),(0,t.jsxs)(eC.Typography.Text,{children:["RPM: ",ek(u)]})]})]})}),(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Total Spend (USD)","Cumulative spend across all budget cycles within this team."),(0,t.jsx)("div",{style:{marginTop:8},children:(0,t.jsxs)(eC.Typography.Title,{level:4,style:{margin:0},children:["$",eN(m,4)]})})]})}),(0,t.jsx)(ev.Col,{xs:24,md:12,children:(0,t.jsxs)(ej.Card,{children:[eT("Model Scope","Models you can access within this team."),(0,t.jsx)("div",{style:{marginTop:8},children:h&&h.length>0?(0,t.jsx)(z.Space,{wrap:!0,children:h.map(e=>(0,t.jsx)(D.Tag,{children:e},e))}):(0,t.jsx)(eC.Typography.Text,{children:"All Team Models"})})]})})]})]})}let eM="overview",eA="my-user",eF="virtual-keys",eO="members",ez="member-permissions",eP="settings",eL={[eM]:"Overview",[eA]:"My User",[eF]:"Virtual Keys",[eO]:"Members",[ez]:"Member Permissions",[eP]:"Settings"};var eD=e.i(292639),eR=e.i(294612);function eE({teamData:e,canEditTeam:a,handleMemberDelete:r,setSelectedEditMember:s,setIsEditMemberModalVisible:i,setIsAddMemberModalVisible:o}){let n=e=>{if(null==e)return"0";if("number"==typeof e){let t=Number(e);return t===Math.floor(t)?t.toString():(0,m.formatNumberWithCommas)(t,8).replace(/\.?0+$/,"")}return"0"},{data:d}=(0,eD.useUISettings)(),{userId:c,userRole:g}=(0,l.default)(),h=!!d?.values?.disable_team_admin_delete_team_user,x=(0,u.isUserTeamAdminForSingleTeam)(e.team_info.members_with_roles,c||""),b=(0,u.isProxyAdminRole)(g||""),_=[{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Model Scope",(0,t.jsx)(R.Tooltip,{title:"Models this member can access. Empty means they inherit all team models.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"model_scope",render:(l,a)=>{let r=(t=>{if(!t)return null;let l=e.team_memberships.find(e=>e.user_id===t),a=l?.litellm_budget_table?.allowed_models;return a&&a.length>0?a:null})(a.user_id);if(!r)return(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"(all team models)"});let s=r.slice(0,2),i=r.length-s.length;return(0,t.jsxs)(z.Space,{wrap:!0,children:[s.map(e=>(0,t.jsx)(eC.Typography.Text,{code:!0,style:{fontSize:"12px"},children:e},e)),i>0&&(0,t.jsx)(R.Tooltip,{title:r.slice(2).join(", "),children:(0,t.jsxs)(eC.Typography.Text,{type:"secondary",children:["+",i," more"]})})]})}},{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Current Cycle Spend (USD)",(0,t.jsx)(R.Tooltip,{title:"Spend for the current budget cycle. Resets to $0 when the member's budget window rolls over. This is the value checked against the member's budget.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"spend",render:(l,a)=>(0,t.jsxs)(eC.Typography.Text,{children:["$",(0,m.formatNumberWithCommas)((t=>{if(!t)return 0;let l=e.team_memberships.find(e=>e.user_id===t);return l?.spend??0})(a.user_id),4)]})},{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Total Spend (USD)",(0,t.jsx)(R.Tooltip,{title:"Cumulative spend by this member within this team, across all budget cycles. Tracking began 2026-04-21; spend from before that date is not included.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"total_spend",render:(l,a)=>(0,t.jsxs)(eC.Typography.Text,{children:["$",(0,m.formatNumberWithCommas)((t=>{if(!t)return 0;let l=e.team_memberships.find(e=>e.user_id===t);return l?.total_spend??0})(a.user_id),4)]})},{title:"Team Member Budget (USD)",key:"budget",render:(l,a)=>{let r=(t=>{if(!t)return null;let l=e.team_memberships.find(e=>e.user_id===t),a=l?.litellm_budget_table?.max_budget;return null==a?null:n(a)})(a.user_id);return(0,t.jsx)(eC.Typography.Text,{children:r?`$${(0,m.formatNumberWithCommas)(Number(r),4)}`:"No Limit"})}},{title:"Budget Reset",key:"budget_reset",render:(l,a)=>{let r=(t=>{if(!t)return null;let l=e.team_memberships.find(e=>e.user_id===t);return ey(l?.litellm_budget_table?.budget_reset_at)})(a.user_id);return r?(0,t.jsx)(eC.Typography.Text,{children:r}):(0,t.jsx)(eC.Typography.Text,{type:"secondary",children:"—"})}},{title:(0,t.jsxs)(z.Space,{direction:"horizontal",children:["Team Member Rate Limits",(0,t.jsx)(R.Tooltip,{title:"Rate limits for this member's usage within this team.",children:(0,t.jsx)(p.InfoCircleOutlined,{})})]}),key:"rate_limits",render:(l,a)=>(0,t.jsx)(eC.Typography.Text,{children:(t=>{if(!t)return"No Limits";let l=e.team_memberships.find(e=>e.user_id===t),a=l?.litellm_budget_table?.rpm_limit,r=l?.litellm_budget_table?.tpm_limit,s=[a?`${n(a)} RPM`:null,r?`${n(r)} TPM`:null].filter(Boolean);return s.length>0?s.join(" / "):"No Limits"})(a.user_id)})}];return(0,t.jsx)(eR.default,{members:e.team_info.members_with_roles,canEdit:a,onEdit:t=>{let l=e.team_memberships.find(e=>e.user_id===t.user_id);s({...t,max_budget_in_team:l?.litellm_budget_table?.max_budget||null,tpm_limit:l?.litellm_budget_table?.tpm_limit||null,rpm_limit:l?.litellm_budget_table?.rpm_limit||null,allowed_models:l?.litellm_budget_table?.allowed_models||[]}),i(!0)},onDelete:r,onAddMember:()=>o(!0),roleColumnTitle:"Team Role",roleTooltip:"This role applies only to this team and is independent from the user's proxy-level role.",extraColumns:_,showDeleteForMember:()=>b||a&&!x||x&&!h})}var eB=e.i(207082),eU=e.i(871943),eV=e.i(502547),eK=e.i(360820),e$=e.i(94629),eG=e.i(152990),eW=e.i(682830),eq=e.i(994388),eH=e.i(752978),eQ=e.i(282786),eY=e.i(981339),eJ=e.i(304911),eX=e.i(969550),eZ=e.i(20147),e0=e.i(633627);function e1({teamId:e,teamAlias:a,organization:r}){let{accessToken:s}=(0,l.default)(),[i,n]=(0,V.useState)(null),[d,c]=(0,V.useState)([{id:"created_at",desc:!0}]),[u,g]=(0,V.useState)({pageIndex:0,pageSize:50}),[h,x]=(0,V.useState)({"Organization ID":"","Key Alias":"","User ID":"","Sort By":"created_at","Sort Order":"desc"}),b=d.length>0?d[0].id:"created_at",_=d.length>0?d[0].desc?"desc":"asc":"desc",f=u.pageIndex,y=u.pageSize,{data:j,isPending:v,isFetching:C,refetch:S}=(0,eB.useKeys)(f+1,y,{teamID:e,organizationID:h["Organization ID"]?.trim()||void 0,selectedKeyAlias:h["Key Alias"]?.trim()||void 0,userID:h["User ID"]?.trim()||void 0,sortBy:b||void 0,sortOrder:_||void 0,expand:"user"}),N=(0,V.useMemo)(()=>{let e=j?.keys||[],t=r?.organization_id;return t?e.map(e=>({...e,organization_id:(e.organization_id??e.org_id)||t})):e},[j?.keys,r?.organization_id]),k=j?.total_pages??0,[I,M]=(0,V.useState)({}),A=(0,V.useMemo)(()=>({team_id:e,team_alias:a||e,models:[],max_budget:null,budget_duration:null,tpm_limit:null,rpm_limit:null,organization_id:r?.organization_id||"",created_at:"",keys:[],members_with_roles:[],spend:0}),[e,a,r]),F=(0,o.useQuery)({queryKey:["teamFilterOptions",e,s],queryFn:async()=>(0,e0.fetchTeamFilterOptions)(s,e),enabled:!!s&&!!e,staleTime:3e4}).data||{keyAliases:[],organizationIds:[],userIds:[]},O=(0,V.useCallback)(()=>{S?.()},[S]);(0,V.useEffect)(()=>(window.addEventListener("storage",O),()=>window.removeEventListener("storage",O)),[O]);let z=(0,V.useCallback)((e,t=!1)=>{x(t=>({...t,"Organization ID":e["Organization ID"]??t["Organization ID"],"Key Alias":e["Key Alias"]??t["Key Alias"],"User ID":e["User ID"]??t["User ID"],"Sort By":e["Sort By"]??t["Sort By"]??"created_at","Sort Order":e["Sort Order"]??t["Sort Order"]??"desc"})),t||g(e=>({...e,pageIndex:0}))},[]),P=(0,V.useCallback)(()=>{x({"Organization ID":"","Key Alias":"","User ID":"","Sort By":"created_at","Sort Order":"desc"}),g(e=>({...e,pageIndex:0}))},[]),L=(0,V.useMemo)(()=>[{name:"Organization ID",label:"Organization ID",isSearchable:!0,searchFn:async e=>{let{organizationIds:t}=F;if(!t.length)return[];let l=e.toLowerCase();return(l?t.filter(e=>e.toLowerCase().includes(l)):t).map(e=>({label:e,value:e}))}},{name:"Key Alias",label:"Key Alias",isSearchable:!0,searchFn:async e=>{let{keyAliases:t}=F,l=e.toLowerCase();return(l?t.filter(e=>e.toLowerCase().includes(l)):t).map(e=>({label:e,value:e}))}},{name:"User ID",label:"User ID",isSearchable:!0,searchFn:async e=>{let{userIds:t}=F,l=e.toLowerCase();return(l?t.filter(e=>e.id.toLowerCase().includes(l)||e.email.toLowerCase().includes(l)):t).map(e=>({label:e.email?`${e.id} (${e.email})`:e.id,value:e.id}))}}],[F]),D=(0,V.useMemo)(()=>[{id:"token",accessorKey:"token",header:"Key ID",size:100,enableSorting:!0,cell:e=>{let l=e.getValue(),a=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:l,children:(0,t.jsx)(eq.Button,{size:"xs",variant:"light",className:"font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate block",style:{maxWidth:a,overflow:"hidden"},onClick:()=>n(e.row.original),children:l??"-"})})}},{id:"key_alias",accessorKey:"key_alias",header:"Key Alias",size:150,enableSorting:!0,cell:e=>{let l=e.getValue(),a=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:l,children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block",style:{maxWidth:a,overflow:"hidden"},children:l??"-"})})}},{id:"key_name",accessorKey:"key_name",header:"Secret Key",size:120,enableSorting:!1,cell:e=>(0,t.jsx)("span",{className:"font-mono text-xs",children:e.getValue()})},{id:"organization_id",accessorKey:"organization_id",header:"Organization ID",size:140,enableSorting:!1,cell:e=>e.getValue()?e.renderValue():"-"},{id:"user_email",accessorKey:"user",header:"User Email",size:160,enableSorting:!1,cell:e=>{let l=e.getValue(),a=l?.user_email,r=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:a,children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block",style:{maxWidth:r,overflow:"hidden"},children:a??"-"})})}},{id:"user_id",accessorKey:"user_id",header:"User ID",size:70,enableSorting:!1,cell:e=>{let l=e.getValue(),a="default_user_id"===l?"Default Proxy Admin":l,r=e.cell.column.getSize();return(0,t.jsx)(R.Tooltip,{title:a,children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block",style:{maxWidth:r,overflow:"hidden"},children:a??"-"})})}},{id:"created_at",accessorKey:"created_at",header:"Created At",size:120,enableSorting:!0,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleDateString():"-"}},{id:"created_by",accessorKey:"created_by",header:"Created By",size:70,enableSorting:!1,cell:e=>{let l=e.getValue();if(!l)return"-";let{created_by_user:a}=e.row.original,r=a?.user_alias??null,s=a?.user_email??null,i="default_user_id"===l,o=r||s||l,n=e.cell.column.getSize(),d=(0,t.jsx)("div",{className:"flex flex-col gap-2 text-xs min-w-[200px] max-w-[300px]",children:[{label:"User Alias",value:r},{label:"User Email",value:s},{label:"User ID",value:l}].map(({label:e,value:l})=>(0,t.jsxs)("div",{className:"flex flex-col min-w-0",children:[(0,t.jsx)("span",{className:"text-gray-400",children:e}),l?(0,t.jsx)(eC.Typography.Text,{className:"font-mono text-xs",ellipsis:{tooltip:l},copyable:!0,children:l}):(0,t.jsx)("span",{className:"font-mono",children:"-"})]},e))});return!i||r||s?(0,t.jsx)(eQ.Popover,{content:d,trigger:"hover",placement:"bottomLeft",children:(0,t.jsx)("span",{className:"font-mono text-xs truncate block cursor-default",style:{maxWidth:n,overflow:"hidden"},children:o})}):(0,t.jsx)(eQ.Popover,{content:d,trigger:"hover",placement:"bottomLeft",children:(0,t.jsx)("span",{className:"cursor-default",children:(0,t.jsx)(eJ.default,{userId:l})})})}},{id:"updated_at",accessorKey:"updated_at",header:"Updated At",size:120,enableSorting:!0,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleDateString():"Never"}},{id:"last_active",accessorKey:"last_active",header:()=>(0,t.jsxs)("span",{className:"flex items-center gap-1",children:["Last Active",(0,t.jsx)(eQ.Popover,{content:"This is a new field and is not backfilled. Only new key usage will update this value.",trigger:"hover",children:(0,t.jsx)(p.InfoCircleOutlined,{className:"text-gray-400 text-xs cursor-help"})})]}),size:130,enableSorting:!1,cell:e=>{let l=e.getValue();if(!l)return"Unknown";let a=new Date(l);return(0,t.jsx)(R.Tooltip,{title:a.toLocaleString(void 0,{dateStyle:"medium",timeStyle:"long"}),children:(0,t.jsx)("span",{children:a.toLocaleDateString()})})}},{id:"expires",accessorKey:"expires",header:"Expires",size:120,enableSorting:!1,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleDateString():"Never"}},{id:"spend",accessorKey:"spend",header:"Spend (USD)",size:100,enableSorting:!0,cell:e=>(0,m.formatNumberWithCommas)(e.getValue(),4)},{id:"max_budget",accessorKey:"max_budget",header:"Budget (USD)",size:110,enableSorting:!0,cell:e=>{let t=e.getValue();return null===t?"Unlimited":`$${(0,m.formatNumberWithCommas)(t)}`}},{id:"budget_reset_at",accessorKey:"budget_reset_at",header:"Budget Reset",size:130,enableSorting:!1,cell:e=>{let t=e.getValue();return t?new Date(t).toLocaleString():"Never"}},{id:"models",accessorKey:"models",header:"Models",size:200,enableSorting:!1,cell:e=>{let l=e.getValue();return(0,t.jsx)("div",{className:"flex flex-col py-2",children:Array.isArray(l)?(0,t.jsx)("div",{className:"flex flex-col",children:0===l.length?(0,t.jsx)(w.Badge,{size:"xs",className:"mb-1",color:"red",children:(0,t.jsx)(T.Text,{children:"All Proxy Models"})}):(0,t.jsx)(t.Fragment,{children:(0,t.jsxs)("div",{className:"flex items-start",children:[l.length>3&&(0,t.jsx)("div",{children:(0,t.jsx)(eH.Icon,{icon:I[e.row.id]?eU.ChevronDownIcon:eV.ChevronRightIcon,className:"cursor-pointer",size:"xs",onClick:()=>M(t=>({...t,[e.row.id]:!t[e.row.id]}))})}),(0,t.jsxs)("div",{className:"flex flex-wrap gap-1",children:[l.slice(0,3).map((e,l)=>"all-proxy-models"===e?(0,t.jsx)(w.Badge,{size:"xs",color:"red",children:(0,t.jsx)(T.Text,{children:"All Proxy Models"})},l):(0,t.jsx)(w.Badge,{size:"xs",color:"blue",children:(0,t.jsx)(T.Text,{children:e.length>30?`${(0,H.getModelDisplayName)(e).slice(0,30)}...`:(0,H.getModelDisplayName)(e)})},l)),l.length>3&&!I[e.row.id]&&(0,t.jsx)(w.Badge,{size:"xs",color:"gray",className:"cursor-pointer",children:(0,t.jsxs)(T.Text,{children:["+",l.length-3," ",l.length-3==1?"more model":"more models"]})}),I[e.row.id]&&(0,t.jsx)("div",{className:"flex flex-wrap gap-1",children:l.slice(3).map((e,l)=>"all-proxy-models"===e?(0,t.jsx)(w.Badge,{size:"xs",color:"red",children:(0,t.jsx)(T.Text,{children:"All Proxy Models"})},l+3):(0,t.jsx)(w.Badge,{size:"xs",color:"blue",children:(0,t.jsx)(T.Text,{children:e.length>30?`${(0,H.getModelDisplayName)(e).slice(0,30)}...`:(0,H.getModelDisplayName)(e)})},l+3))})]})]})})}):null})}},{id:"rate_limits",header:"Rate Limits",size:140,enableSorting:!1,cell:({row:e})=>{let l=e.original;return(0,t.jsxs)("div",{children:[(0,t.jsxs)("div",{children:["TPM: ",null!==l.tpm_limit?l.tpm_limit:"Unlimited"]}),(0,t.jsxs)("div",{children:["RPM: ",null!==l.rpm_limit?l.rpm_limit:"Unlimited"]})]})}}],[I]),E=(0,V.useCallback)(e=>{let t="function"==typeof e?e(d):e;if(c(t),t?.length>0){let e=t[0];z({"Sort By":e.id,"Sort Order":e.desc?"desc":"asc"},!0)}},[d,z]),B=(0,eG.useReactTable)({data:N,columns:D,columnResizeMode:"onChange",columnResizeDirection:"ltr",state:{sorting:d,pagination:u},onSortingChange:E,onPaginationChange:g,getCoreRowModel:(0,eW.getCoreRowModel)(),enableSorting:!0,manualSorting:!0,manualPagination:!0,pageCount:k});return(0,t.jsx)("div",{className:"w-full h-full overflow-hidden",children:i?(0,t.jsx)(eZ.default,{keyId:i.token,onClose:()=>n(null),keyData:i,teams:[A],onDelete:S}):(0,t.jsxs)("div",{className:"border-b py-4 flex-1 overflow-hidden",children:[(0,t.jsx)("div",{className:"w-full mb-6",children:(0,t.jsx)(eX.default,{options:L,onApplyFilters:z,initialValues:h,onResetFilters:P})}),(0,t.jsx)("div",{className:"flex items-center justify-end w-full mb-4",children:(0,t.jsxs)("div",{className:"inline-flex items-center gap-2",children:[v||C?(0,t.jsx)(eY.Skeleton.Node,{active:!0,style:{width:74,height:20}}):(0,t.jsxs)("span",{className:"text-sm text-gray-700",children:["Page ",f+1," of ",B.getPageCount()]}),v||C?(0,t.jsx)(eY.Skeleton.Button,{active:!0,size:"small",style:{width:84,height:30}}):(0,t.jsx)("button",{onClick:()=>B.previousPage(),disabled:v||C||!B.getCanPreviousPage(),className:"px-3 py-1 text-sm border rounded-md hover:bg-gray-50 disabled:opacity-50 disabled:cursor-not-allowed",children:"Previous"}),v||C?(0,t.jsx)(eY.Skeleton.Button,{active:!0,size:"small",style:{width:58,height:30}}):(0,t.jsx)("button",{onClick:()=>B.nextPage(),disabled:v||C||!B.getCanNextPage(),className:"px-3 py-1 text-sm border rounded-md hover:bg-gray-50 disabled:opacity-50 disabled:cursor-not-allowed",children:"Next"})]})}),(0,t.jsx)("div",{className:"h-[75vh] overflow-auto",children:(0,t.jsx)("div",{className:"rounded-lg custom-border relative",children:(0,t.jsx)("div",{className:"overflow-x-auto",children:(0,t.jsxs)(ed.Table,{className:"[&_td]:py-0.5 [&_th]:py-1",style:{width:B.getCenterTotalSize()},children:[(0,t.jsx)(eu.TableHead,{children:B.getHeaderGroups().map(e=>(0,t.jsx)(eh.TableRow,{children:e.headers.map(e=>(0,t.jsx)(eg.TableHeaderCell,{"data-header-id":e.id,className:`py-1 h-8 relative hover:bg-gray-50 ${"actions"===e.id?"sticky right-0 bg-white shadow-[-4px_0_8px_-6px_rgba(0,0,0,0.1)]":""}`,style:{width:e.getSize(),position:"relative",cursor:e.column.getCanSort()?"pointer":"default"},onMouseEnter:()=>{let t=document.querySelector(`[data-header-id="${e.id}"] .resizer`);t&&(t.style.opacity="0.5")},onMouseLeave:()=>{let t=document.querySelector(`[data-header-id="${e.id}"] .resizer`);t&&!e.column.getIsResizing()&&(t.style.opacity="0")},onClick:e.column.getCanSort()?e.column.getToggleSortingHandler():void 0,children:(0,t.jsxs)("div",{className:"flex items-center justify-between gap-2",children:[(0,t.jsx)("div",{className:"flex items-center",children:e.isPlaceholder?null:(0,eG.flexRender)(e.column.columnDef.header,e.getContext())}),"actions"!==e.id&&e.column.getCanSort()&&(0,t.jsx)("div",{className:"w-4",children:e.column.getIsSorted()?({asc:(0,t.jsx)(eK.ChevronUpIcon,{className:"h-4 w-4 text-blue-500"}),desc:(0,t.jsx)(eU.ChevronDownIcon,{className:"h-4 w-4 text-blue-500"})})[e.column.getIsSorted()]:(0,t.jsx)(e$.SwitchVerticalIcon,{className:"h-4 w-4 text-gray-400"})}),(0,t.jsx)("div",{onDoubleClick:()=>e.column.resetSize(),onMouseDown:e.getResizeHandler(),onTouchStart:e.getResizeHandler(),className:`resizer ${B.options.columnResizeDirection} ${e.column.getIsResizing()?"isResizing":""}`,style:{position:"absolute",right:0,top:0,height:"100%",width:"5px",background:e.column.getIsResizing()?"#3b82f6":"transparent",cursor:"col-resize",userSelect:"none",touchAction:"none",opacity:+!!e.column.getIsResizing()}})]})},e.id))},e.id))}),(0,t.jsx)(em.TableBody,{children:v||C?(0,t.jsx)(eh.TableRow,{children:(0,t.jsx)(ec.TableCell,{colSpan:D.length,className:"h-8 text-center",children:(0,t.jsx)("div",{className:"text-center text-gray-500",children:(0,t.jsx)("p",{children:"Loading keys..."})})})}):N.length>0?B.getRowModel().rows.map(e=>(0,t.jsx)(eh.TableRow,{className:"h-8",children:e.getVisibleCells().map(e=>(0,t.jsx)(ec.TableCell,{style:{width:e.column.getSize(),maxWidth:"8-x",whiteSpace:"pre-wrap",overflow:"hidden"},className:`py-0.5 max-h-8 overflow-hidden text-ellipsis whitespace-nowrap ${"models"===e.column.id&&Array.isArray(e.getValue())&&e.getValue().length>3?"px-0":""}`,children:(0,eG.flexRender)(e.column.columnDef.cell,e.getContext())},e.id))},e.id)):(0,t.jsx)(eh.TableRow,{children:(0,t.jsx)(ec.TableCell,{colSpan:D.length,className:"h-8 text-center",children:(0,t.jsx)("div",{className:"text-center text-gray-500",children:(0,t.jsx)("p",{children:"No keys found"})})})})})]})})})})]})})}e.s(["default",0,({teamId:e,onClose:o,accessToken:n,is_team_admin:en,is_proxy_admin:ed,is_org_admin:em=!1,userModels:ec,editTeam:eu,premiumUser:eg=!1,onUpdate:eh})=>{let ep,ex,eb,ef,ey,ej,[ev,ew]=(0,V.useState)(null),[eC,eS]=(0,V.useState)(!0),[eT,eN]=(0,V.useState)(!1),[ek]=M.Form.useForm(),[eD,eR]=(0,V.useState)(!1),[eB,eU]=(0,V.useState)(null),[eV,eK]=(0,V.useState)(!1),[e$,eG]=(0,V.useState)([]),[eW,eq]=(0,V.useState)(!1),[eH,eQ]=(0,V.useState)({}),{data:eY,isLoading:eJ}=d(),eX=eY?.globalGuardrailNames??new Set,[eZ,e0]=(0,V.useState)([]),[e2,e4]=(0,V.useState)({}),[e5,e3]=(0,V.useState)(!1),[e7,e6]=(0,V.useState)(null),[e9,e8]=(0,V.useState)(!1),[te,tt]=(0,V.useState)(!1),[tl,ta]=(0,V.useState)(!1),tr=V.default.useRef(null),[ts,ti]=(0,V.useState)(null),{userRole:to,userId:tn}=(0,l.default)(),{data:td=[]}=(0,a.useOrganizations)(),tm=(0,r.useQueryClient)(),tc=(0,V.useMemo)(()=>{let e=ev?.team_info?.organization_id;if(!e||!tn)return!1;let t=td.find(t=>t.organization_id===e);return t?.members?.some(e=>e.user_id===tn&&"org_admin"===e.user_role)??!1},[ev,td,tn]),tu=M.Form.useWatch("models",ek),tg=M.Form.useWatch("disable_global_guardrails",ek),th=(0,V.useMemo)(()=>{let e=tu??ev?.team_info?.models??[];return e.includes("all-proxy-models")||e.includes("all-team-models")?ec:(0,H.unfurlWildcardModelsInList)(e,ec)},[tu,ev,ec]),tp=en||ed||em||tc,tx=(0,V.useMemo)(()=>{let e;return e=[eM,eA,eF],tp?[...e,eO,ez,eP]:e},[tp]),tb=(0,V.useMemo)(()=>eu&&tp?eP:eM,[eu,tp]),t_=async()=>{try{if(eS(!0),!n)return;let t=await (0,i.teamInfoCall)(n,e);ew(t)}catch(e){ee.default.fromBackend("Failed to load team information"),console.error("Error fetching team info:",e)}finally{eS(!1)}};(0,V.useEffect)(()=>{t_()},[e,n]),(0,V.useEffect)(()=>{(async()=>{if(!n||!ev?.team_info?.organization_id)return ti(null);try{let e=await (0,i.organizationInfoCall)(n,ev.team_info.organization_id);ti(e)}catch(e){console.error("Error fetching organization info:",e),ti(null)}})()},[n,ev?.team_info?.organization_id]),(0,V.useMemo)(()=>{let e;return e=[],e=ts?ts.models.includes("all-proxy-models")?ec:ts.models.length>0?ts.models:ec:ec,(0,H.unfurlWildcardModelsInList)(e,ec)},[ts,ec]),(0,V.useEffect)(()=>{(async()=>{try{if(!n)return;let e=(await (0,i.getPoliciesList)(n)).policies.map(e=>e.policy_name);e0(e)}catch(e){console.error("Failed to fetch policies:",e)}})()},[n]),(0,V.useEffect)(()=>{(async()=>{if(!n||!ev?.team_info?.policies||0===ev.team_info.policies.length)return;e3(!0);let e={};try{await Promise.all(ev.team_info.policies.map(async t=>{try{let l=await (0,i.getPolicyInfoWithGuardrails)(n,t);e[t]=l.resolved_guardrails||[]}catch(l){console.error(`Failed to fetch guardrails for policy ${t}:`,l),e[t]=[]}})),e4(e)}catch(e){console.error("Failed to fetch policy guardrails:",e)}finally{e3(!1)}})()},[n,ev?.team_info?.policies]);let tf=async t=>{try{if(null==n)return;let l={user_email:t.user_email,user_id:t.user_id,role:t.role};await (0,i.teamMemberAddCall)(n,e,l),ee.default.success("Team member added successfully"),eN(!1),ek.resetFields();let a=await (0,i.teamInfoCall)(n,e);ew(a),eh(a)}catch(t){let e="Failed to add team member";t?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")?e="Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this.":t?.message&&(e=t.message),ee.default.fromBackend(e),console.error("Error adding team member:",t)}},ty=async t=>{try{if(null==n)return;let l={user_email:t.user_email,user_id:t.user_id,role:t.role,max_budget_in_team:t.max_budget_in_team,tpm_limit:t.tpm_limit,rpm_limit:t.rpm_limit,allowed_models:t.allowed_models};E.default.destroy(),await (0,i.teamMemberUpdateCall)(n,e,l),ee.default.success("Team member updated successfully"),eR(!1);let a=await (0,i.teamInfoCall)(n,e);ew(a),eh(a)}catch(t){let e="Failed to update team member";t?.raw?.detail?.includes("Assigning team admins is a premium feature")?e="Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this.":t?.message&&(e=t.message),eR(!1),E.default.destroy(),ee.default.fromBackend(e),console.error("Error updating team member:",t)}},tj=async()=>{if(e7&&n){tt(!0);try{await (0,i.teamMemberDeleteCall)(n,e,e7),ee.default.success("Team member removed successfully");let t=await (0,i.teamInfoCall)(n,e);ew(t),eh(t)}catch(e){ee.default.fromBackend("Failed to remove team member"),console.error("Error removing team member:",e)}finally{tt(!1),e8(!1),e6(null)}}},tv=async t=>{try{let l;if(!n)return;ta(!0);let r={};try{let{soft_budget_alerting_emails:e,...l}=t.metadata?JSON.parse(t.metadata):{};r=l}catch(e){ee.default.fromBackend("Invalid JSON in metadata field");return}if("string"==typeof t.secret_manager_settings&&t.secret_manager_settings.trim().length>0)try{l=JSON.parse(t.secret_manager_settings)}catch(e){ee.default.fromBackend("Invalid JSON in secret manager settings");return}let s=e=>null==e||"string"==typeof e&&""===e.trim()||"number"==typeof e&&Number.isNaN(e)?null:e,o={},d={};for(let e of t.modelLimits??[])e?.model&&(null!=e.tpm&&(o[e.model]=e.tpm),null!=e.rpm&&(d[e.model]=e.rpm));let m=!0===t.disable_global_guardrails,u=m?Array.from(eX):Array.from(eX).filter(e=>!(t.guardrails||[]).includes(e)),g=ed?{allowed_passthrough_routes:t.allowed_passthrough_routes||[]}:tw.metadata?.allowed_passthrough_routes?{allowed_passthrough_routes:tw.metadata.allowed_passthrough_routes}:{},h={team_id:e,team_alias:t.team_alias,models:t.models,tpm_limit:s(t.tpm_limit),rpm_limit:s(t.rpm_limit),model_tpm_limit:o,model_rpm_limit:d,max_budget:t.max_budget,soft_budget:s(t.soft_budget),budget_duration:t.budget_duration,metadata:{...r,...g,guardrails:(t.guardrails||[]).filter(e=>!eX.has(e)),opted_out_global_guardrails:u,...t.logging_settings?.length>0?{logging:t.logging_settings}:{},disable_global_guardrails:m,soft_budget_alerting_emails:"string"==typeof t.soft_budget_alerting_emails?t.soft_budget_alerting_emails.split(",").map(e=>e.trim()).filter(e=>e.length>0):t.soft_budget_alerting_emails||[],...void 0!==l?{secret_manager_settings:l}:{}},...t.policies?.length>0?{policies:t.policies}:{},...t.organization_id!==tw.organization_id?{organization_id:t.organization_id??null}:{}};h.max_budget=(0,c.mapEmptyStringToNull)(h.max_budget),h.team_member_budget_duration=t.team_member_budget_duration,void 0!==t.team_member_budget&&(h.team_member_budget=Number(t.team_member_budget)),void 0!==t.team_member_key_duration&&(h.team_member_key_duration=t.team_member_key_duration),(void 0!==t.team_member_tpm_limit||void 0!==t.team_member_rpm_limit)&&(h.team_member_tpm_limit=s(t.team_member_tpm_limit),h.team_member_rpm_limit=s(t.team_member_rpm_limit));let{servers:p,accessGroups:x,toolsets:b}=t.mcp_servers_and_groups||{servers:[],accessGroups:[],toolsets:[]},_=new Set(p||[]),f=Object.fromEntries(Object.entries(t.mcp_tool_permissions||{}).filter(([e])=>_.has(e)));h.object_permission={},p&&(h.object_permission.mcp_servers=p),x&&(h.object_permission.mcp_access_groups=x),f&&(h.object_permission.mcp_tool_permissions=f),b&&(h.object_permission.mcp_toolsets=b),delete t.mcp_servers_and_groups,delete t.mcp_tool_permissions;let{agents:y,accessGroups:j}=t.agents_and_groups||{agents:[],accessGroups:[]};y&&y.length>0&&(h.object_permission.agents=y),j&&j.length>0&&(h.object_permission.agent_access_groups=j),delete t.agents_and_groups,t.vector_stores&&t.vector_stores.length>0&&(h.object_permission.vector_stores=t.vector_stores),Array.isArray(t.object_permission_search_tools)&&(h.object_permission.search_tools=t.object_permission_search_tools),void 0!==t.access_group_ids&&(h.access_group_ids=t.access_group_ids),void 0!==t.default_team_member_models&&(h.default_team_member_models=t.default_team_member_models);let v=tr.current?.getValue();if(v?.router_settings){let e=e=>null!=e&&""!==e&&!1!==e&&!(Array.isArray(e)&&0===e.length),t=Object.values(v.router_settings).some(e),l=tw.router_settings&&Object.values(tw.router_settings).some(e);(t||l)&&(h.router_settings=v.router_settings)}await (0,i.teamUpdateCall)(n,h),tm.invalidateQueries({queryKey:a.organizationKeys.all}),ee.default.success("Team settings updated successfully"),eK(!1),t_()}catch(e){console.error("Error updating team:",e)}finally{ta(!1)}};if(eC)return(0,t.jsx)("div",{className:"p-4",children:"Loading..."});if(!ev?.team_info)return(0,t.jsx)("div",{className:"p-4",children:"Team not found"});let{team_info:tw}=ev,tC=tw.metadata?.disable_global_guardrails===!0,tS=new Set(Array.isArray(tw.metadata?.opted_out_global_guardrails)?tw.metadata.opted_out_global_guardrails:[]),tT=(Array.isArray(tw.metadata?.guardrails)?tw.metadata.guardrails:[]).filter(e=>!eX.has(e)),tN=tC?tT:[...Array.from(eX).filter(e=>!tS.has(e)),...tT],tk=e=>{e.preventDefault(),e.stopPropagation()},tI=async(e,t)=>{await (0,m.copyToClipboard)(e)&&(eQ(e=>({...e,[t]:!0})),setTimeout(()=>{eQ(e=>({...e,[t]:!1}))},2e3))};return(0,t.jsxs)("div",{className:"p-4",children:[(0,t.jsx)("div",{className:"flex justify-between items-center mb-6",children:(0,t.jsxs)("div",{children:[(0,t.jsx)(I.Button,{type:"text",icon:(0,t.jsx)(f.ArrowLeftIcon,{className:"h-4 w-4"}),onClick:o,className:"mb-4",children:"Back to Teams"}),(0,t.jsx)(k.Title,{children:tw.team_alias}),(0,t.jsxs)("div",{className:"flex items-center",children:[(0,t.jsx)(T.Text,{className:"text-gray-500 font-mono",children:tw.team_id}),(0,t.jsx)(I.Button,{type:"text",size:"small",icon:eH["team-id"]?(0,t.jsx)(B.CheckIcon,{size:12}):(0,t.jsx)(U.CopyIcon,{size:12}),onClick:()=>tI(tw.team_id,"team-id"),className:`left-2 z-10 transition-all duration-200 ${eH["team-id"]?"text-green-600 bg-green-50 border-green-200":"text-gray-500 hover:text-gray-700 hover:bg-gray-100"}`})]})]})}),(0,t.jsx)(L.Tabs,{defaultActiveKey:tb,className:"mb-4",items:[{key:eM,label:eL[eM],children:(0,t.jsxs)(S.Grid,{numItems:1,numItemsSm:2,numItemsLg:3,className:"gap-6",children:[(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{children:"Budget Status"}),(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsxs)(k.Title,{children:["$",(0,m.formatNumberWithCommas)(tw.spend,4)]}),(0,t.jsxs)(T.Text,{children:["of ",null===tw.max_budget?"Unlimited":`$${(0,m.formatNumberWithCommas)(tw.max_budget,4)}`]}),tw.budget_duration&&(0,t.jsxs)(T.Text,{className:"text-gray-500",children:["Reset: ",tw.budget_duration]}),(0,t.jsx)("br",{}),tw.team_member_budget_table&&(0,t.jsxs)(T.Text,{className:"text-gray-500",children:["Team Member Budget: $",(0,m.formatNumberWithCommas)(tw.team_member_budget_table.max_budget,4)]})]})]}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{children:"Rate Limits"}),(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsxs)(T.Text,{children:["TPM: ",tw.tpm_limit||"Unlimited"]}),(0,t.jsxs)(T.Text,{children:["RPM: ",tw.rpm_limit||"Unlimited"]}),tw.max_parallel_requests&&(0,t.jsxs)(T.Text,{children:["Max Parallel Requests: ",tw.max_parallel_requests]}),(ep=tw.metadata?.model_tpm_limit??{},ex=tw.metadata?.model_rpm_limit??{},0===(eb=Array.from(new Set([...Object.keys(ep),...Object.keys(ex)]))).length?null:(0,t.jsxs)("div",{className:"mt-3",children:[(0,t.jsx)(T.Text,{className:"text-gray-500",children:"Per-model limits:"}),eb.map(e=>(0,t.jsxs)(T.Text,{className:"text-xs",children:[e,": TPM ",ep[e]??"—",", RPM ",ex[e]??"—"]},e))]}))]})]}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{children:"Models"}),(0,t.jsx)("div",{className:"mt-2 flex flex-wrap gap-2",children:0===tw.models.length||tw.models.includes("all-proxy-models")?(0,t.jsx)(w.Badge,{color:"red",children:"All proxy models"}):(0,t.jsxs)(t.Fragment,{children:[tw.models.map((e,l)=>(0,t.jsx)(w.Badge,{color:"blue",children:e},`direct-${l}`)),(tw.access_group_models||[]).map((e,l)=>(0,t.jsx)(w.Badge,{color:"green",title:"From access group",children:e},`ag-${l}`))]})})]}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{className:"font-semibold text-gray-900",children:"Virtual Keys"}),(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsxs)(T.Text,{children:["User Keys: ",ev.keys.filter(e=>e.user_id).length]}),(0,t.jsxs)(T.Text,{children:["Service Account Keys: ",ev.keys.filter(e=>!e.user_id).length]}),(0,t.jsxs)(T.Text,{className:"text-gray-500",children:["Total: ",ev.keys.length]})]})]}),(0,t.jsx)(et.default,{objectPermission:tw.object_permission,variant:"card",accessToken:n}),(0,t.jsx)(C.Card,{children:(0,t.jsx)(Q,{globalGuardrailNames:eX,teamGuardrails:Array.isArray(tw.metadata?.guardrails)?tw.metadata.guardrails:[],optedOutGlobalGuardrails:Array.isArray(tw.metadata?.opted_out_global_guardrails)?tw.metadata.opted_out_global_guardrails:[],killSwitchOn:tC,variant:"inline"})}),(0,t.jsxs)(C.Card,{children:[(0,t.jsx)(T.Text,{className:"font-semibold text-gray-900 mb-3",children:"Policies"}),tw.policies&&tw.policies.length>0?(0,t.jsx)("div",{className:"space-y-4",children:tw.policies.map((e,l)=>(0,t.jsxs)("div",{className:"space-y-2",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)(w.Badge,{color:"purple",children:e}),e5&&(0,t.jsx)(T.Text,{className:"text-xs text-gray-400",children:"Loading guardrails..."})]}),!e5&&e2[e]&&e2[e].length>0&&(0,t.jsxs)("div",{className:"ml-4 pl-3 border-l-2 border-gray-200",children:[(0,t.jsx)(T.Text,{className:"text-xs text-gray-500 mb-1",children:"Resolved Guardrails:"}),(0,t.jsx)("div",{className:"flex flex-wrap gap-1",children:e2[e].map((e,l)=>(0,t.jsx)(w.Badge,{color:"blue",size:"xs",children:e},l))})]})]},l))}):(0,t.jsx)(T.Text,{className:"text-gray-500",children:"No policies configured"})]}),(0,t.jsx)(Y.default,{loggingConfigs:tw.metadata?.logging||[],disabledCallbacks:[],variant:"card"})]})},{key:eA,label:eL[eA],children:(0,t.jsx)(eI,{teamId:e})},{key:eF,label:eL[eF],children:(0,t.jsx)(e1,{teamId:e,teamAlias:tw.team_alias,organization:ts})},{key:eO,label:eL[eO],children:(0,t.jsx)(eE,{teamData:ev,canEditTeam:tp,handleMemberDelete:e=>{e6(e),e8(!0)},setSelectedEditMember:eU,setIsEditMemberModalVisible:eR,setIsAddMemberModalVisible:eN})},{key:ez,label:eL[ez],children:(0,t.jsx)(e_,{teamId:e,accessToken:n,canEditTeam:tp})},{key:eP,label:eL[eP],children:(0,t.jsxs)(C.Card,{className:"overflow-y-auto max-h-[65vh]",children:[(0,t.jsxs)("div",{className:"flex justify-between items-center mb-4",children:[(0,t.jsx)(k.Title,{children:"Team Settings"}),tp&&!eV&&(0,t.jsx)(I.Button,{icon:(0,t.jsx)(g.EditOutlined,{className:"h-4 w-4"}),onClick:()=>eK(!0),children:"Edit Settings"})]}),eV&&eJ?(0,t.jsx)("div",{className:"p-4",children:"Loading..."}):eV?(0,t.jsxs)(M.Form,{form:ek,onFinish:tv,onValuesChange:e=>{if("disable_global_guardrails"in e){let t=!0===e.disable_global_guardrails,l=(ek.getFieldValue("guardrails")||[]).filter(e=>!eX.has(e));ek.setFieldValue("guardrails",t?l:[...Array.from(eX),...l])}},initialValues:{...tw,team_alias:tw.team_alias,models:tw.models,tpm_limit:tw.tpm_limit,rpm_limit:tw.rpm_limit,object_permission_search_tools:tw.object_permission?.search_tools||[],modelLimits:Array.from(new Set([...Object.keys(tw.metadata?.model_tpm_limit??{}),...Object.keys(tw.metadata?.model_rpm_limit??{})])).map(e=>({model:e,tpm:tw.metadata?.model_tpm_limit?.[e],rpm:tw.metadata?.model_rpm_limit?.[e]})),max_budget:tw.max_budget,soft_budget:tw.soft_budget,budget_duration:tw.budget_duration,team_member_tpm_limit:tw.team_member_budget_table?.tpm_limit,team_member_rpm_limit:tw.team_member_budget_table?.rpm_limit,team_member_budget:tw.team_member_budget_table?.max_budget,team_member_budget_duration:tw.team_member_budget_table?.budget_duration,guardrails:tN,policies:tw.policies||[],disable_global_guardrails:tw.metadata?.disable_global_guardrails||!1,soft_budget_alerting_emails:Array.isArray(tw.metadata?.soft_budget_alerting_emails)?tw.metadata.soft_budget_alerting_emails.join(", "):"",metadata:tw.metadata?JSON.stringify((({logging:e,secret_manager_settings:t,soft_budget_alerting_emails:l,model_tpm_limit:a,model_rpm_limit:r,allowed_passthrough_routes:s,...i})=>i)(tw.metadata),null,2):"",logging_settings:tw.metadata?.logging||[],secret_manager_settings:tw.metadata?.secret_manager_settings?JSON.stringify(tw.metadata.secret_manager_settings,null,2):"",organization_id:tw.organization_id,vector_stores:tw.object_permission?.vector_stores||[],mcp_servers:tw.object_permission?.mcp_servers||[],mcp_access_groups:tw.object_permission?.mcp_access_groups||[],mcp_servers_and_groups:{servers:tw.object_permission?.mcp_servers||[],accessGroups:tw.object_permission?.mcp_access_groups||[],toolsets:tw.object_permission?.mcp_toolsets||[]},mcp_tool_permissions:tw.object_permission?.mcp_tool_permissions||{},agents_and_groups:{agents:tw.object_permission?.agents||[],accessGroups:tw.object_permission?.agent_access_groups||[]},access_group_ids:tw.access_group_ids||[],default_team_member_models:tw.default_team_member_models||[],allowed_passthrough_routes:tw.metadata?.allowed_passthrough_routes||[]},layout:"vertical",children:[(0,t.jsx)(M.Form.Item,{label:"Team Name",name:"team_alias",rules:[{required:!0,message:"Please input a team name"}],children:(0,t.jsx)(A.Input,{type:""})}),(0,t.jsx)(M.Form.Item,{label:"Models",name:"models",rules:[{required:!0,message:"Please select at least one model"}],children:(0,t.jsx)(Z.ModelSelect,{value:ek.getFieldValue("models")||[],onChange:e=>ek.setFieldValue("models",e),teamID:e,organizationID:ev?.team_info?.organization_id||void 0,options:{includeSpecialOptions:!0,includeUserModels:!ev?.team_info?.organization_id,showAllProxyModelsOverride:(0,u.isProxyAdminRole)(to)&&!ev?.team_info?.organization_id},context:"team",dataTestId:"models-select"})}),(0,t.jsx)(M.Form.Item,{label:"Max Budget (USD)",name:"max_budget",children:(0,t.jsx)(el.default,{step:.01,precision:2,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Soft Budget (USD)",name:"soft_budget",children:(0,t.jsx)(el.default,{step:.01,precision:2,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Soft Budget Alerting Emails",name:"soft_budget_alerting_emails",tooltip:"Comma-separated email addresses to receive alerts when the soft budget is reached",children:(0,t.jsx)(A.Input,{placeholder:"example1@test.com, example2@test.com"})}),(0,t.jsxs)(y.Accordion,{className:"mt-4 mb-4",children:[(0,t.jsx)(v.AccordionHeader,{children:(0,t.jsx)("b",{children:"Team Member Settings"})}),(0,t.jsxs)(j.AccordionBody,{children:[(0,t.jsx)(T.Text,{className:"text-xs text-gray-500 mb-4",children:"Optional defaults applied when members join this team. All fields can be overridden per member."}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Default Model Access"," ",(0,t.jsx)(R.Tooltip,{title:"Optional. If set, new members can only access these models by default. Must be a subset of the team's models above. Leave empty to give all members access to all team models.",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),name:"default_team_member_models",children:(0,t.jsx)(M.Form.Item,{noStyle:!0,shouldUpdate:(e,t)=>e.models!==t.models,children:({getFieldValue:e})=>{let l=e("models")||tw.models||[];return(0,t.jsx)(O.Select,{mode:"multiple",placeholder:"Leave empty — all team models accessible to every member",value:ek.getFieldValue("default_team_member_models")||[],onChange:e=>ek.setFieldValue("default_team_member_models",e),options:l.map(e=>({label:e,value:e}))})}})}),(0,t.jsx)(M.Form.Item,{label:"Default Budget (USD)",name:"team_member_budget",tooltip:"Default spend budget for each member in this team.",children:(0,t.jsx)(el.default,{step:.01,precision:2,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Default Budget Duration",name:"team_member_budget_duration",children:(0,t.jsx)(W,{onChange:e=>ek.setFieldValue("team_member_budget_duration",e),value:ek.getFieldValue("team_member_budget_duration")})}),(0,t.jsx)(M.Form.Item,{label:"Default Key Duration (eg: 1d, 1mo)",name:"team_member_key_duration",tooltip:"Set a limit to the duration of a team member's key. Format: 30s (seconds), 30m (minutes), 30h (hours), 30d (days), 1mo (month)",children:(0,t.jsx)(N.TextInput,{placeholder:"e.g., 30d"})}),(0,t.jsx)(M.Form.Item,{label:"Default TPM Limit",name:"team_member_tpm_limit",tooltip:"Default tokens per minute limit for each member. Can be overridden per member.",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"},placeholder:"e.g., 1000"})}),(0,t.jsx)(M.Form.Item,{label:"Default RPM Limit",name:"team_member_rpm_limit",tooltip:"Default requests per minute limit for each member. Can be overridden per member.",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"},placeholder:"e.g., 100"})})]})]}),(0,t.jsx)(M.Form.Item,{label:"Reset Budget",name:"budget_duration",children:(0,t.jsxs)(O.Select,{placeholder:"n/a",children:[(0,t.jsx)(O.Select.Option,{value:"24h",children:"daily"}),(0,t.jsx)(O.Select.Option,{value:"7d",children:"weekly"}),(0,t.jsx)(O.Select.Option,{value:"30d",children:"monthly"})]})}),(0,t.jsx)(M.Form.Item,{label:"Tokens per minute Limit (TPM)",name:"tpm_limit",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Requests per minute Limit (RPM)",name:"rpm_limit",children:(0,t.jsx)(el.default,{step:1,style:{width:"100%"}})}),(0,t.jsx)(M.Form.Item,{label:"Model-Specific Rate Limits",tooltip:"Set per-model TPM/RPM limits that apply across the whole team.",children:(0,t.jsx)(M.Form.List,{name:"modelLimits",children:(e,{add:l,remove:a})=>(0,t.jsxs)(t.Fragment,{children:[e.map(({key:e,name:l,...r})=>(0,t.jsxs)(z.Space,{style:{display:"flex",marginBottom:8},align:"baseline",children:[(0,t.jsx)(M.Form.Item,{...r,name:[l,"model"],rules:[{required:!0,message:"Missing model"},{validator:(e,t)=>t&&(ek.getFieldValue("modelLimits")??[]).filter(e=>e?.model===t).length>1?Promise.reject(Error("Duplicate model")):Promise.resolve()}],style:{minWidth:240},children:(0,t.jsx)(O.Select,{showSearch:!0,placeholder:"Select model",allowClear:!0,options:th.map(e=>({value:e,label:e}))})}),(0,t.jsx)(M.Form.Item,{...r,name:[l,"tpm"],rules:[{validator:async(e,t)=>{let a=(ek.getFieldValue("modelLimits")??[])[l]??{};return a.model&&null==t&&null==a.rpm?Promise.reject(Error("Set at least one of TPM or RPM")):Promise.resolve()}}],children:(0,t.jsx)(F.InputNumber,{placeholder:"TPM Limit",min:0})}),(0,t.jsx)(M.Form.Item,{...r,name:[l,"rpm"],children:(0,t.jsx)(F.InputNumber,{placeholder:"RPM Limit",min:0})}),(0,t.jsx)(x.MinusCircleOutlined,{onClick:()=>a(l),style:{color:"#ef4444"}})]},e)),(0,t.jsx)(M.Form.Item,{children:(0,t.jsx)(I.Button,{type:"dashed",onClick:()=>l(),block:!0,icon:(0,t.jsx)(b.PlusOutlined,{}),children:"Add Model Limit"})})]})})}),(0,t.jsx)(M.Form.Item,{label:"Router Settings",children:(0,t.jsx)(ei.default,{ref:tr,accessToken:n||"",value:tw.router_settings?{router_settings:tw.router_settings}:void 0})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Guardrails"," ",(0,t.jsx)(R.Tooltip,{title:"Select which guardrails apply to this team. Global guardrails are enabled by default — uncheck to opt out. Other guardrails are opt-in.",children:(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/guardrails/quick_start",target:"_blank",rel:"noopener noreferrer",onClick:e=>e.stopPropagation(),children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})})]}),name:"guardrails",children:(0,t.jsxs)(O.Select,{mode:"multiple",placeholder:"Select guardrails",optionLabelProp:"label",tagRender:({label:e,value:l,closable:a,onClose:r})=>{let s=eX.has(l);return(0,t.jsxs)(D.Tag,{color:"blue",closable:a,onClose:r,onMouseDown:tk,style:{marginInlineEnd:4},children:[s&&(0,t.jsx)(h.GlobalOutlined,{style:{marginInlineEnd:4},"aria-label":"Global guardrail"}),e]})},children:[(0,t.jsx)(O.Select.OptGroup,{label:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(h.GlobalOutlined,{style:{marginInlineEnd:4}}),"Global"]}),children:(eY?.guardrails??[]).filter(e=>e.litellm_params?.default_on).map(e=>(0,t.jsx)(O.Select.Option,{value:e.guardrail_name,label:e.guardrail_name,disabled:tg,children:e.guardrail_name},e.guardrail_name))}),(0,t.jsx)(O.Select.OptGroup,{label:"Other",children:(eY?.guardrails??[]).filter(e=>!e.litellm_params?.default_on).map(e=>(0,t.jsx)(O.Select.Option,{value:e.guardrail_name,label:e.guardrail_name,children:e.guardrail_name},e.guardrail_name))})]})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Disable all global guardrails"," ",(0,t.jsx)(R.Tooltip,{title:"Kill switch: bypass every global guardrail for this team, including any added in the future. For per-guardrail opt-out instead, use the Guardrails dropdown above.",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),name:"disable_global_guardrails",valuePropName:"checked",children:(0,t.jsx)(P.Switch,{checkedChildren:"Yes",unCheckedChildren:"No"})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Policies"," ",(0,t.jsx)(R.Tooltip,{title:"Apply policies to this team to control guardrails and other settings",children:(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies",target:"_blank",rel:"noopener noreferrer",onClick:e=>e.stopPropagation(),children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})})]}),name:"policies",children:(0,t.jsx)(O.Select,{mode:"tags",placeholder:"Select or enter policies",options:eZ.map(e=>({value:e,label:e}))})}),(0,t.jsx)(M.Form.Item,{label:(0,t.jsxs)("span",{children:["Access Groups"," ",(0,t.jsx)(R.Tooltip,{title:"Assign access groups to this team. Access groups control which models, MCP servers, and agents this team can use",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),name:"access_group_ids",children:(0,t.jsx)(K.default,{placeholder:"Select access groups (optional)"})}),(0,t.jsx)(M.Form.Item,{label:"Vector Stores",name:"vector_stores","aria-label":"Vector Stores",children:(0,t.jsx)(ea.default,{onChange:e=>ek.setFieldValue("vector_stores",e),value:ek.getFieldValue("vector_stores"),accessToken:n||"",placeholder:"Select vector stores"})}),(0,t.jsx)(M.Form.Item,{label:"Allowed Pass Through Routes",name:"allowed_passthrough_routes",children:(0,t.jsx)(R.Tooltip,{title:eg?ed?"":"Only proxy admins can set allowed pass through routes":"Premium feature - Upgrade to set allowed pass through routes",placement:"top",children:(0,t.jsx)(q.default,{onChange:e=>ek.setFieldValue("allowed_passthrough_routes",e),value:ek.getFieldValue("allowed_passthrough_routes"),accessToken:n||"",placeholder:"Select pass through routes",disabled:!eg||!ed})})}),(0,t.jsx)(M.Form.Item,{label:"MCP Servers / Access Groups",name:"mcp_servers_and_groups",children:(0,t.jsx)(J.default,{onChange:e=>ek.setFieldValue("mcp_servers_and_groups",e),value:ek.getFieldValue("mcp_servers_and_groups"),accessToken:n||"",placeholder:"Select MCP servers or access groups (optional)"})}),(0,t.jsx)(M.Form.Item,{name:"mcp_tool_permissions",initialValue:{},hidden:!0,children:(0,t.jsx)(A.Input,{type:"hidden"})}),(0,t.jsx)(M.Form.Item,{noStyle:!0,shouldUpdate:(e,t)=>e.mcp_servers_and_groups!==t.mcp_servers_and_groups||e.mcp_tool_permissions!==t.mcp_tool_permissions,children:()=>(0,t.jsx)("div",{className:"mb-6",children:(0,t.jsx)(X.default,{accessToken:n||"",selectedServers:ek.getFieldValue("mcp_servers_and_groups")?.servers||[],toolPermissions:ek.getFieldValue("mcp_tool_permissions")||{},onChange:e=>ek.setFieldsValue({mcp_tool_permissions:e})})})}),(0,t.jsx)(M.Form.Item,{label:"Agents / Access Groups",name:"agents_and_groups",children:(0,t.jsx)($.default,{onChange:e=>ek.setFieldValue("agents_and_groups",e),value:ek.getFieldValue("agents_and_groups"),accessToken:n||"",placeholder:"Select agents or access groups (optional)"})}),(0,t.jsxs)(y.Accordion,{className:"mt-4 mb-4",children:[(0,t.jsx)(v.AccordionHeader,{children:(0,t.jsx)("b",{children:"Search Tool Settings"})}),(0,t.jsx)(j.AccordionBody,{children:(0,t.jsx)(M.Form.Item,{label:"Allowed Search Tools",name:"object_permission_search_tools",tooltip:"Select which search tools this team can access. Leave empty to allow all search tools.",children:(0,t.jsx)(er,{onChange:e=>ek.setFieldValue("object_permission_search_tools",e),value:ek.getFieldValue("object_permission_search_tools"),accessToken:n||"",placeholder:"Select search tools (optional, empty = all allowed)"})})})]}),(0,t.jsx)(M.Form.Item,{label:"Organization",name:"organization_id",children:(0,t.jsx)(O.Select,{allowClear:!0,placeholder:"Select an organization",showSearch:!0,optionFilterProp:"label",options:td.map(e=>({value:e.organization_id,label:e.organization_alias||e.organization_id}))})}),(0,t.jsx)(M.Form.Item,{label:"Logging Settings",name:"logging_settings",children:(0,t.jsx)(es.default,{value:ek.getFieldValue("logging_settings"),onChange:e=>ek.setFieldValue("logging_settings",e)})}),(0,t.jsx)(M.Form.Item,{label:"Secret Manager Settings",name:"secret_manager_settings",help:eg?"Enter secret manager configuration as a JSON object.":"Premium feature - Upgrade to manage secret manager settings.",rules:[{validator:async(e,t)=>{if(!t)return Promise.resolve();try{return JSON.parse(t),Promise.resolve()}catch(e){return Promise.reject(Error("Please enter valid JSON"))}}}],children:(0,t.jsx)(A.Input.TextArea,{rows:6,placeholder:'{"namespace": "admin", "mount": "secret", "path_prefix": "litellm"}',disabled:!eg})}),(0,t.jsx)(M.Form.Item,{label:"Metadata",name:"metadata",children:(0,t.jsx)(A.Input.TextArea,{rows:10})}),(0,t.jsx)("div",{className:"sticky z-10 bg-white p-4 pr-0 border-t border-gray-200 bottom-[-1.5rem] inset-x-[-1.5rem]",children:(0,t.jsxs)("div",{className:"flex justify-end items-center gap-2",children:[(0,t.jsx)(I.Button,{onClick:()=>eK(!1),disabled:tl,children:"Cancel"}),(0,t.jsx)(I.Button,{icon:(0,t.jsx)(_.SaveOutlined,{className:"h-4 w-4"}),type:"primary",htmlType:"submit",loading:tl,children:"Save Changes"})]})})]}):(0,t.jsxs)("div",{className:"space-y-4",children:[(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Team Name"}),(0,t.jsx)("div",{children:tw.team_alias})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Team ID"}),(0,t.jsx)("div",{className:"font-mono",children:tw.team_id})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Created At"}),(0,t.jsx)("div",{children:new Date(tw.created_at).toLocaleString()})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Models"}),(0,t.jsx)("div",{className:"flex flex-wrap gap-2 mt-1",children:tw.models.map((e,l)=>(0,t.jsx)(w.Badge,{color:"red",children:e},l))})]}),tw.default_team_member_models&&tw.default_team_member_models.length>0&&(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Default Member Models"}),(0,t.jsx)("div",{className:"flex flex-wrap gap-2 mt-1",children:tw.default_team_member_models.map((e,l)=>(0,t.jsx)(w.Badge,{color:"blue",children:e},l))})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Rate Limits"}),(0,t.jsxs)("div",{children:["TPM: ",tw.tpm_limit||"Unlimited"]}),(0,t.jsxs)("div",{children:["RPM: ",tw.rpm_limit||"Unlimited"]}),(ef=tw.metadata?.model_tpm_limit??{},ey=tw.metadata?.model_rpm_limit??{},0===(ej=Array.from(new Set([...Object.keys(ef),...Object.keys(ey)]))).length?null:(0,t.jsxs)("div",{className:"mt-2",children:[(0,t.jsx)(T.Text,{className:"text-gray-500",children:"Per-model limits:"}),ej.map(e=>(0,t.jsxs)("div",{className:"text-xs ml-2",children:[e,": TPM ",ef[e]??"—",", RPM ",ey[e]??"—"]},e))]}))]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Team Budget"}),(0,t.jsxs)("div",{children:["Max Budget:"," ",null!==tw.max_budget?`$${(0,m.formatNumberWithCommas)(tw.max_budget,4)}`:"No Limit"]}),(0,t.jsxs)("div",{children:["Soft Budget:"," ",null!==tw.soft_budget&&void 0!==tw.soft_budget?`$${(0,m.formatNumberWithCommas)(tw.soft_budget,4)}`:"No Limit"]}),(0,t.jsxs)("div",{children:["Budget Reset: ",tw.budget_duration||"Never"]}),tw.metadata?.soft_budget_alerting_emails&&Array.isArray(tw.metadata.soft_budget_alerting_emails)&&tw.metadata.soft_budget_alerting_emails.length>0&&(0,t.jsxs)("div",{children:["Soft Budget Alerting Emails: ",tw.metadata.soft_budget_alerting_emails.join(", ")]})]}),(0,t.jsxs)("div",{children:[(0,t.jsxs)(T.Text,{className:"font-medium",children:["Team Member Settings"," ",(0,t.jsx)(R.Tooltip,{title:"These are limits on individual team members",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),(0,t.jsxs)("div",{children:["Max Budget: ",tw.team_member_budget_table?.max_budget||"No Limit"]}),(0,t.jsxs)("div",{children:["Budget Duration: ",tw.team_member_budget_table?.budget_duration||"No Limit"]}),(0,t.jsxs)("div",{children:["Key Duration: ",tw.metadata?.team_member_key_duration||"No Limit"]}),(0,t.jsxs)("div",{children:["TPM Limit: ",tw.team_member_budget_table?.tpm_limit||"No Limit"]}),(0,t.jsxs)("div",{children:["RPM Limit: ",tw.team_member_budget_table?.rpm_limit||"No Limit"]})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Router Settings"}),tw.router_settings&&Object.values(tw.router_settings).some(e=>null!=e&&""!==e&&!(Array.isArray(e)&&0===e.length))?(0,t.jsxs)("div",{className:"mt-1 space-y-1",children:[tw.router_settings.routing_strategy&&(0,t.jsxs)("div",{children:["Routing Strategy:"," ",(0,t.jsx)(w.Badge,{color:"blue",children:tw.router_settings.routing_strategy})]}),null!=tw.router_settings.num_retries&&(0,t.jsxs)("div",{children:["Number of Retries: ",tw.router_settings.num_retries]}),null!=tw.router_settings.allowed_fails&&(0,t.jsxs)("div",{children:["Allowed Failures: ",tw.router_settings.allowed_fails]}),null!=tw.router_settings.cooldown_time&&(0,t.jsxs)("div",{children:["Cooldown Time: ",tw.router_settings.cooldown_time,"s"]}),null!=tw.router_settings.timeout&&(0,t.jsxs)("div",{children:["Timeout: ",tw.router_settings.timeout,"s"]}),null!=tw.router_settings.retry_after&&(0,t.jsxs)("div",{children:["Retry After: ",tw.router_settings.retry_after,"s"]}),tw.router_settings.fallbacks&&Array.isArray(tw.router_settings.fallbacks)&&tw.router_settings.fallbacks.length>0&&(0,t.jsxs)("div",{children:["Fallbacks: ",tw.router_settings.fallbacks.length," configured"]}),tw.router_settings.enable_tag_filtering&&(0,t.jsx)("div",{children:"Tag Filtering: Enabled"})]}):(0,t.jsx)("div",{className:"text-gray-400",children:"No router settings configured"})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Organization ID"}),(0,t.jsx)("div",{children:tw.organization_id})]}),(0,t.jsxs)("div",{children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Status"}),(0,t.jsx)(w.Badge,{color:tw.blocked?"red":"green",children:tw.blocked?"Blocked":"Active"})]}),(0,t.jsx)(et.default,{objectPermission:tw.object_permission,variant:"inline",className:"pt-4 border-t border-gray-200",accessToken:n}),(0,t.jsx)(Q,{globalGuardrailNames:eX,teamGuardrails:Array.isArray(tw.metadata?.guardrails)?tw.metadata.guardrails:[],optedOutGlobalGuardrails:Array.isArray(tw.metadata?.opted_out_global_guardrails)?tw.metadata.opted_out_global_guardrails:[],killSwitchOn:tC,variant:"inline",className:"pt-4 border-t border-gray-200"}),(0,t.jsx)(Y.default,{loggingConfigs:tw.metadata?.logging||[],disabledCallbacks:[],variant:"inline",className:"pt-4 border-t border-gray-200"}),tw.metadata?.secret_manager_settings&&(0,t.jsxs)("div",{className:"pt-4 border-t border-gray-200",children:[(0,t.jsx)(T.Text,{className:"font-medium",children:"Secret Manager Settings"}),(0,t.jsx)("pre",{className:"mt-2 bg-gray-50 p-3 rounded text-xs overflow-x-auto",children:JSON.stringify(tw.metadata.secret_manager_settings,null,2)})]})]})]})}].filter(e=>tx.includes(e.key))}),(0,t.jsx)(eo.default,{visible:eD,onCancel:()=>eR(!1),onSubmit:ty,initialData:eB,mode:"edit",config:{title:"Edit Member",showEmail:!0,showUserId:!0,roleOptions:[{label:"Admin",value:"admin"},{label:"User",value:"user"}],additionalFields:[{name:"max_budget_in_team",label:(0,t.jsxs)("span",{children:["Team Member Budget (USD)"," ",(0,t.jsx)(R.Tooltip,{title:"Maximum amount in USD this member can spend within this team. This is separate from any global user budget limits",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"numerical",step:.01,min:0,placeholder:"Budget limit for this member within this team"},{name:"tpm_limit",label:(0,t.jsxs)("span",{children:["Team Member TPM Limit"," ",(0,t.jsx)(R.Tooltip,{title:"Maximum tokens per minute this member can use within this team. This is separate from any global user TPM limit",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"numerical",step:1,min:0,placeholder:"Tokens per minute limit for this member in this team"},{name:"rpm_limit",label:(0,t.jsxs)("span",{children:["Team Member RPM Limit"," ",(0,t.jsx)(R.Tooltip,{title:"Maximum requests per minute this member can make within this team. This is separate from any global user RPM limit",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"numerical",step:1,min:0,placeholder:"Requests per minute limit for this member in this team"},{name:"allowed_models",label:(0,t.jsxs)("span",{children:["Allowed Models"," ",(0,t.jsx)(R.Tooltip,{title:"Models this member can access within this team. Leave empty to inherit all team models.",children:(0,t.jsx)(p.InfoCircleOutlined,{style:{marginLeft:"4px"}})})]}),type:"multi-select",options:(tw.models||[]).map(e=>({label:e,value:e})),placeholder:"Leave empty to inherit all team models"}]}}),(0,t.jsx)(s.default,{isVisible:eT,onCancel:()=>eN(!1),onSubmit:tf,accessToken:n,teamId:e}),(0,t.jsx)(G.default,{isOpen:e9,title:"Delete Team Member",alertMessage:"Removing team members will also delete any keys created by or created for this member.",message:"Are you sure you want to remove this member from the team? This action cannot be undone.",resourceInformationTitle:"Team Member Information",resourceInformation:[{label:"User ID",value:e7?.user_id,code:!0},{label:"Email",value:e7?.user_email},{label:"Role",value:e7?.role}],onCancel:()=>{e8(!1),e6(null)},onOk:tj,confirmLoading:te})]})}],56567)}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/048f065ef4eab631.js b/litellm/proxy/_experimental/out/_next/static/chunks/048f065ef4eab631.js new file mode 100644 index 000000000000..2a53043e9348 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/048f065ef4eab631.js @@ -0,0 +1 @@ +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,888288,e=>{"use strict";var t=e.i(271645);let r=(e,r)=>{let a=void 0!==r,[n,l]=(0,t.useState)(e);return[a?r:n,e=>{a||l(e)}]};e.s(["default",()=>r])},757440,e=>{"use strict";var t=e.i(290571),r=e.i(271645);let a=e=>{var a=(0,t.__rest)(e,[]);return r.default.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},a),r.default.createElement("path",{d:"M11.9999 13.1714L16.9497 8.22168L18.3639 9.63589L11.9999 15.9999L5.63599 9.63589L7.0502 8.22168L11.9999 13.1714Z"}))};e.s(["default",()=>a])},446428,854056,e=>{"use strict";let t;var r=e.i(290571),a=e.i(271645);let n=e=>{var t=(0,r.__rest)(e,[]);return a.default.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},t),a.default.createElement("path",{d:"M12 22C6.47715 22 2 17.5228 2 12C2 6.47715 6.47715 2 12 2C17.5228 2 22 6.47715 22 12C22 17.5228 17.5228 22 12 22ZM12 10.5858L9.17157 7.75736L7.75736 9.17157L10.5858 12L7.75736 14.8284L9.17157 16.2426L12 13.4142L14.8284 16.2426L16.2426 14.8284L13.4142 12L16.2426 9.17157L14.8284 7.75736L12 10.5858Z"}))};e.s(["default",()=>n],446428);var l=e.i(746725),s=e.i(914189),i=e.i(553521),o=e.i(835696),u=e.i(941444),d=e.i(178677),c=e.i(294316),m=e.i(83733),h=e.i(233137),f=e.i(732607),p=e.i(397701),g=e.i(700020);function v(e){var t;return!!(e.enter||e.enterFrom||e.enterTo||e.leave||e.leaveFrom||e.leaveTo)||(null!=(t=e.as)?t:k)!==a.Fragment||1===a.default.Children.count(e.children)}let b=(0,a.createContext)(null);b.displayName="TransitionContext";var x=((t=x||{}).Visible="visible",t.Hidden="hidden",t);let w=(0,a.createContext)(null);function y(e){return"children"in e?y(e.children):e.current.filter(({el:e})=>null!==e.current).filter(({state:e})=>"visible"===e).length>0}function C(e,t){let r=(0,u.useLatestValue)(e),n=(0,a.useRef)([]),o=(0,i.useIsMounted)(),d=(0,l.useDisposables)(),c=(0,s.useEvent)((e,t=g.RenderStrategy.Hidden)=>{let a=n.current.findIndex(({el:t})=>t===e);-1!==a&&((0,p.match)(t,{[g.RenderStrategy.Unmount](){n.current.splice(a,1)},[g.RenderStrategy.Hidden](){n.current[a].state="hidden"}}),d.microTask(()=>{var e;!y(n)&&o.current&&(null==(e=r.current)||e.call(r))}))}),m=(0,s.useEvent)(e=>{let t=n.current.find(({el:t})=>t===e);return t?"visible"!==t.state&&(t.state="visible"):n.current.push({el:e,state:"visible"}),()=>c(e,g.RenderStrategy.Unmount)}),h=(0,a.useRef)([]),f=(0,a.useRef)(Promise.resolve()),v=(0,a.useRef)({enter:[],leave:[]}),b=(0,s.useEvent)((e,r,a)=>{h.current.splice(0),t&&(t.chains.current[r]=t.chains.current[r].filter(([t])=>t!==e)),null==t||t.chains.current[r].push([e,new Promise(e=>{h.current.push(e)})]),null==t||t.chains.current[r].push([e,new Promise(e=>{Promise.all(v.current[r].map(([e,t])=>t)).then(()=>e())})]),"enter"===r?f.current=f.current.then(()=>null==t?void 0:t.wait.current).then(()=>a(r)):a(r)}),x=(0,s.useEvent)((e,t,r)=>{Promise.all(v.current[t].splice(0).map(([e,t])=>t)).then(()=>{var e;null==(e=h.current.shift())||e()}).then(()=>r(t))});return(0,a.useMemo)(()=>({children:n,register:m,unregister:c,onStart:b,onStop:x,wait:f,chains:v}),[m,c,n,b,x,v,f])}w.displayName="NestingContext";let k=a.Fragment,M=g.RenderFeatures.RenderStrategy,S=(0,g.forwardRefWithAs)(function(e,t){let{show:r,appear:n=!1,unmount:l=!0,...i}=e,u=(0,a.useRef)(null),m=v(e),f=(0,c.useSyncRefs)(...m?[u,t]:null===t?[]:[t]);(0,d.useServerHandoffComplete)();let p=(0,h.useOpenClosed)();if(void 0===r&&null!==p&&(r=(p&h.State.Open)===h.State.Open),void 0===r)throw Error("A is used but it is missing a `show={true | false}` prop.");let[x,k]=(0,a.useState)(r?"visible":"hidden"),S=C(()=>{r||k("hidden")}),[E,N]=(0,a.useState)(!0),T=(0,a.useRef)([r]);(0,o.useIsoMorphicEffect)(()=>{!1!==E&&T.current[T.current.length-1]!==r&&(T.current.push(r),N(!1))},[T,r]);let $=(0,a.useMemo)(()=>({show:r,appear:n,initial:E}),[r,n,E]);(0,o.useIsoMorphicEffect)(()=>{r?k("visible"):y(S)||null===u.current||k("hidden")},[r,S]);let O={unmount:l},_=(0,s.useEvent)(()=>{var t;E&&N(!1),null==(t=e.beforeEnter)||t.call(e)}),I=(0,s.useEvent)(()=>{var t;E&&N(!1),null==(t=e.beforeLeave)||t.call(e)}),L=(0,g.useRender)();return a.default.createElement(w.Provider,{value:S},a.default.createElement(b.Provider,{value:$},L({ourProps:{...O,as:a.Fragment,children:a.default.createElement(j,{ref:f,...O,...i,beforeEnter:_,beforeLeave:I})},theirProps:{},defaultTag:a.Fragment,features:M,visible:"visible"===x,name:"Transition"})))}),j=(0,g.forwardRefWithAs)(function(e,t){var r,n;let{transition:l=!0,beforeEnter:i,afterEnter:u,beforeLeave:x,afterLeave:S,enter:j,enterFrom:E,enterTo:N,entered:T,leave:$,leaveFrom:O,leaveTo:_,...I}=e,[L,A]=(0,a.useState)(null),D=(0,a.useRef)(null),R=v(e),F=(0,c.useSyncRefs)(...R?[D,t,A]:null===t?[]:[t]),P=null==(r=I.unmount)||r?g.RenderStrategy.Unmount:g.RenderStrategy.Hidden,{show:z,appear:H,initial:U}=function(){let e=(0,a.useContext)(b);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),[V,B]=(0,a.useState)(z?"visible":"hidden"),W=function(){let e=(0,a.useContext)(w);if(null===e)throw Error("A is used but it is missing a parent or .");return e}(),{register:Y,unregister:K}=W;(0,o.useIsoMorphicEffect)(()=>Y(D),[Y,D]),(0,o.useIsoMorphicEffect)(()=>{if(P===g.RenderStrategy.Hidden&&D.current)return z&&"visible"!==V?void B("visible"):(0,p.match)(V,{hidden:()=>K(D),visible:()=>Y(D)})},[V,D,Y,K,z,P]);let q=(0,d.useServerHandoffComplete)();(0,o.useIsoMorphicEffect)(()=>{if(R&&q&&"visible"===V&&null===D.current)throw Error("Did you forget to passthrough the `ref` to the actual DOM node?")},[D,V,q,R]);let Q=U&&!H,Z=H&&z&&U,X=(0,a.useRef)(!1),J=C(()=>{X.current||(B("hidden"),K(D))},W),G=(0,s.useEvent)(e=>{X.current=!0,J.onStart(D,e?"enter":"leave",e=>{"enter"===e?null==i||i():"leave"===e&&(null==x||x())})}),ee=(0,s.useEvent)(e=>{let t=e?"enter":"leave";X.current=!1,J.onStop(D,t,e=>{"enter"===e?null==u||u():"leave"===e&&(null==S||S())}),"leave"!==t||y(J)||(B("hidden"),K(D))});(0,a.useEffect)(()=>{R&&l||(G(z),ee(z))},[z,R,l]);let et=!(!l||!R||!q||Q),[,er]=(0,m.useTransition)(et,L,z,{start:G,end:ee}),ea=(0,g.compact)({ref:F,className:(null==(n=(0,f.classNames)(I.className,Z&&j,Z&&E,er.enter&&j,er.enter&&er.closed&&E,er.enter&&!er.closed&&N,er.leave&&$,er.leave&&!er.closed&&O,er.leave&&er.closed&&_,!er.transition&&z&&T))?void 0:n.trim())||void 0,...(0,m.transitionDataAttributes)(er)}),en=0;"visible"===V&&(en|=h.State.Open),"hidden"===V&&(en|=h.State.Closed),er.enter&&(en|=h.State.Opening),er.leave&&(en|=h.State.Closing);let el=(0,g.useRender)();return a.default.createElement(w.Provider,{value:J},a.default.createElement(h.OpenClosedProvider,{value:en},el({ourProps:ea,theirProps:I,defaultTag:k,features:M,visible:"visible"===V,name:"Transition.Child"})))}),E=(0,g.forwardRefWithAs)(function(e,t){let r=null!==(0,a.useContext)(b),n=null!==(0,h.useOpenClosed)();return a.default.createElement(a.default.Fragment,null,!r&&n?a.default.createElement(S,{ref:t,...e}):a.default.createElement(j,{ref:t,...e}))}),N=Object.assign(S,{Child:E,Root:S});e.s(["Transition",()=>N],854056)},206929,e=>{"use strict";var t=e.i(290571),r=e.i(757440),a=e.i(271645),n=e.i(446428),l=e.i(444755),s=e.i(673706),i=e.i(103471),o=e.i(495470),u=e.i(854056),d=e.i(888288);let c=(0,s.makeClassName)("Select"),m=a.default.forwardRef((e,s)=>{let{defaultValue:m="",value:h,onValueChange:f,placeholder:p="Select...",disabled:g=!1,icon:v,enableClear:b=!1,required:x,children:w,name:y,error:C=!1,errorMessage:k,className:M,id:S}=e,j=(0,t.__rest)(e,["defaultValue","value","onValueChange","placeholder","disabled","icon","enableClear","required","children","name","error","errorMessage","className","id"]),E=(0,a.useRef)(null),N=a.Children.toArray(w),[T,$]=(0,d.default)(m,h),O=(0,a.useMemo)(()=>{let e=a.default.Children.toArray(w).filter(a.isValidElement);return(0,i.constructValueToNameMapping)(e)},[w]);return a.default.createElement("div",{className:(0,l.tremorTwMerge)("w-full min-w-[10rem] text-tremor-default",M)},a.default.createElement("div",{className:"relative"},a.default.createElement("select",{title:"select-hidden",required:x,className:(0,l.tremorTwMerge)("h-full w-full absolute left-0 top-0 -z-10 opacity-0"),value:T,onChange:e=>{e.preventDefault()},name:y,disabled:g,id:S,onFocus:()=>{let e=E.current;e&&e.focus()}},a.default.createElement("option",{className:"hidden",value:"",disabled:!0,hidden:!0},p),N.map(e=>{let t=e.props.value,r=e.props.children;return a.default.createElement("option",{className:"hidden",key:t,value:t},r)})),a.default.createElement(o.Listbox,Object.assign({as:"div",ref:s,defaultValue:T,value:T,onChange:e=>{null==f||f(e),$(e)},disabled:g,id:S},j),({value:e})=>{var t;return a.default.createElement(a.default.Fragment,null,a.default.createElement(o.ListboxButton,{ref:E,className:(0,l.tremorTwMerge)("w-full outline-none text-left whitespace-nowrap truncate rounded-tremor-default focus:ring-2 transition duration-100 border pr-8 py-2","border-tremor-border shadow-tremor-input focus:border-tremor-brand-subtle focus:ring-tremor-brand-muted","dark:border-dark-tremor-border dark:shadow-dark-tremor-input dark:focus:border-dark-tremor-brand-subtle dark:focus:ring-dark-tremor-brand-muted",v?"pl-10":"pl-3",(0,i.getSelectButtonColors)((0,i.hasValue)(e),g,C))},v&&a.default.createElement("span",{className:(0,l.tremorTwMerge)("absolute inset-y-0 left-0 flex items-center ml-px pl-2.5")},a.default.createElement(v,{className:(0,l.tremorTwMerge)(c("Icon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),a.default.createElement("span",{className:"w-[90%] block truncate"},e&&null!=(t=O.get(e))?t:p),a.default.createElement("span",{className:(0,l.tremorTwMerge)("absolute inset-y-0 right-0 flex items-center mr-3")},a.default.createElement(r.default,{className:(0,l.tremorTwMerge)(c("arrowDownIcon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}))),b&&T?a.default.createElement("button",{type:"button",className:(0,l.tremorTwMerge)("absolute inset-y-0 right-0 flex items-center mr-8"),onClick:e=>{e.preventDefault(),$(""),null==f||f("")}},a.default.createElement(n.default,{className:(0,l.tremorTwMerge)(c("clearIcon"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})):null,a.default.createElement(u.Transition,{enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},a.default.createElement(o.ListboxOptions,{anchor:"bottom start",className:(0,l.tremorTwMerge)("z-10 w-[var(--button-width)] divide-y overflow-y-auto outline-none rounded-tremor-default max-h-[228px] border [--anchor-gap:4px]","bg-tremor-background border-tremor-border divide-tremor-border shadow-tremor-dropdown","dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border dark:shadow-dark-tremor-dropdown")},w)))})),C&&k?a.default.createElement("p",{className:(0,l.tremorTwMerge)("errorMessage","text-sm text-rose-500 mt-1")},k):null)});m.displayName="Select",e.s(["Select",()=>m],206929)},160818,e=>{"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M854.4 800.9c.2-.3.5-.6.7-.9C920.6 722.1 960 621.7 960 512s-39.4-210.1-104.8-288c-.2-.3-.5-.5-.7-.8-1.1-1.3-2.1-2.5-3.2-3.7-.4-.5-.8-.9-1.2-1.4l-4.1-4.7-.1-.1c-1.5-1.7-3.1-3.4-4.6-5.1l-.1-.1c-3.2-3.4-6.4-6.8-9.7-10.1l-.1-.1-4.8-4.8-.3-.3c-1.5-1.5-3-2.9-4.5-4.3-.5-.5-1-1-1.6-1.5-1-1-2-1.9-3-2.8-.3-.3-.7-.6-1-1C736.4 109.2 629.5 64 512 64s-224.4 45.2-304.3 119.2c-.3.3-.7.6-1 1-1 .9-2 1.9-3 2.9-.5.5-1 1-1.6 1.5-1.5 1.4-3 2.9-4.5 4.3l-.3.3-4.8 4.8-.1.1c-3.3 3.3-6.5 6.7-9.7 10.1l-.1.1c-1.6 1.7-3.1 3.4-4.6 5.1l-.1.1c-1.4 1.5-2.8 3.1-4.1 4.7-.4.5-.8.9-1.2 1.4-1.1 1.2-2.1 2.5-3.2 3.7-.2.3-.5.5-.7.8C103.4 301.9 64 402.3 64 512s39.4 210.1 104.8 288c.2.3.5.6.7.9l3.1 3.7c.4.5.8.9 1.2 1.4l4.1 4.7c0 .1.1.1.1.2 1.5 1.7 3 3.4 4.6 5l.1.1c3.2 3.4 6.4 6.8 9.6 10.1l.1.1c1.6 1.6 3.1 3.2 4.7 4.7l.3.3c3.3 3.3 6.7 6.5 10.1 9.6 80.1 74 187 119.2 304.5 119.2s224.4-45.2 304.3-119.2a300 300 0 0010-9.6l.3-.3c1.6-1.6 3.2-3.1 4.7-4.7l.1-.1c3.3-3.3 6.5-6.7 9.6-10.1l.1-.1c1.5-1.7 3.1-3.3 4.6-5 0-.1.1-.1.1-.2 1.4-1.5 2.8-3.1 4.1-4.7.4-.5.8-.9 1.2-1.4a99 99 0 003.3-3.7zm4.1-142.6c-13.8 32.6-32 62.8-54.2 90.2a444.07 444.07 0 00-81.5-55.9c11.6-46.9 18.8-98.4 20.7-152.6H887c-3 40.9-12.6 80.6-28.5 118.3zM887 484H743.5c-1.9-54.2-9.1-105.7-20.7-152.6 29.3-15.6 56.6-34.4 81.5-55.9A373.86 373.86 0 01887 484zM658.3 165.5c39.7 16.8 75.8 40 107.6 69.2a394.72 394.72 0 01-59.4 41.8c-15.7-45-35.8-84.1-59.2-115.4 3.7 1.4 7.4 2.9 11 4.4zm-90.6 700.6c-9.2 7.2-18.4 12.7-27.7 16.4V697a389.1 389.1 0 01115.7 26.2c-8.3 24.6-17.9 47.3-29 67.8-17.4 32.4-37.8 58.3-59 75.1zm59-633.1c11 20.6 20.7 43.3 29 67.8A389.1 389.1 0 01540 327V141.6c9.2 3.7 18.5 9.1 27.7 16.4 21.2 16.7 41.6 42.6 59 75zM540 640.9V540h147.5c-1.6 44.2-7.1 87.1-16.3 127.8l-.3 1.2A445.02 445.02 0 00540 640.9zm0-156.9V383.1c45.8-2.8 89.8-12.5 130.9-28.1l.3 1.2c9.2 40.7 14.7 83.5 16.3 127.8H540zm-56 56v100.9c-45.8 2.8-89.8 12.5-130.9 28.1l-.3-1.2c-9.2-40.7-14.7-83.5-16.3-127.8H484zm-147.5-56c1.6-44.2 7.1-87.1 16.3-127.8l.3-1.2c41.1 15.6 85 25.3 130.9 28.1V484H336.5zM484 697v185.4c-9.2-3.7-18.5-9.1-27.7-16.4-21.2-16.7-41.7-42.7-59.1-75.1-11-20.6-20.7-43.3-29-67.8 37.2-14.6 75.9-23.3 115.8-26.1zm0-370a389.1 389.1 0 01-115.7-26.2c8.3-24.6 17.9-47.3 29-67.8 17.4-32.4 37.8-58.4 59.1-75.1 9.2-7.2 18.4-12.7 27.7-16.4V327zM365.7 165.5c3.7-1.5 7.3-3 11-4.4-23.4 31.3-43.5 70.4-59.2 115.4-21-12-40.9-26-59.4-41.8 31.8-29.2 67.9-52.4 107.6-69.2zM165.5 365.7c13.8-32.6 32-62.8 54.2-90.2 24.9 21.5 52.2 40.3 81.5 55.9-11.6 46.9-18.8 98.4-20.7 152.6H137c3-40.9 12.6-80.6 28.5-118.3zM137 540h143.5c1.9 54.2 9.1 105.7 20.7 152.6a444.07 444.07 0 00-81.5 55.9A373.86 373.86 0 01137 540zm228.7 318.5c-39.7-16.8-75.8-40-107.6-69.2 18.5-15.8 38.4-29.7 59.4-41.8 15.7 45 35.8 84.1 59.2 115.4-3.7-1.4-7.4-2.9-11-4.4zm292.6 0c-3.7 1.5-7.3 3-11 4.4 23.4-31.3 43.5-70.4 59.2-115.4 21 12 40.9 26 59.4 41.8a373.81 373.81 0 01-107.6 69.2z"}}]},name:"global",theme:"outlined"};var n=e.i(9583),l=r.forwardRef(function(e,l){return r.createElement(n.default,(0,t.default)({},e,{ref:l,icon:a}))});e.s(["GlobalOutlined",0,l],160818)},822315,(e,t,r)=>{e.e,t.exports=function(){"use strict";var e="millisecond",t="second",r="minute",a="hour",n="week",l="month",s="quarter",i="year",o="date",u="Invalid Date",d=/^(\d{4})[-/]?(\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/,c=/\[([^\]]+)]|Y{1,4}|M{1,4}|D{1,2}|d{1,4}|H{1,2}|h{1,2}|a|A|m{1,2}|s{1,2}|Z{1,2}|SSS/g,m=function(e,t,r){var a=String(e);return!a||a.length>=t?e:""+Array(t+1-a.length).join(r)+e},h="en",f={};f[h]={name:"en",weekdays:"Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),months:"January_February_March_April_May_June_July_August_September_October_November_December".split("_"),ordinal:function(e){var t=["th","st","nd","rd"],r=e%100;return"["+e+(t[(r-20)%10]||t[r]||t[0])+"]"}};var p="$isDayjsObject",g=function(e){return e instanceof w||!(!e||!e[p])},v=function e(t,r,a){var n;if(!t)return h;if("string"==typeof t){var l=t.toLowerCase();f[l]&&(n=l),r&&(f[l]=r,n=l);var s=t.split("-");if(!n&&s.length>1)return e(s[0])}else{var i=t.name;f[i]=t,n=i}return!a&&n&&(h=n),n||!a&&h},b=function(e,t){if(g(e))return e.clone();var r="object"==typeof t?t:{};return r.date=e,r.args=arguments,new w(r)},x={s:m,z:function(e){var t=-e.utcOffset(),r=Math.abs(t);return(t<=0?"+":"-")+m(Math.floor(r/60),2,"0")+":"+m(r%60,2,"0")},m:function e(t,r){if(t.date(){"use strict";e.i(247167);var t=e.i(931067),r=e.i(271645);let a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M909.1 209.3l-56.4 44.1C775.8 155.1 656.2 92 521.9 92 290 92 102.3 279.5 102 511.5 101.7 743.7 289.8 932 521.9 932c181.3 0 335.8-115 394.6-276.1 1.5-4.2-.7-8.9-4.9-10.3l-56.7-19.5a8 8 0 00-10.1 4.8c-1.8 5-3.8 10-5.9 14.9-17.3 41-42.1 77.8-73.7 109.4A344.77 344.77 0 01655.9 829c-42.3 17.9-87.4 27-133.8 27-46.5 0-91.5-9.1-133.8-27A341.5 341.5 0 01279 755.2a342.16 342.16 0 01-73.7-109.4c-17.9-42.4-27-87.4-27-133.9s9.1-91.5 27-133.9c17.3-41 42.1-77.8 73.7-109.4 31.6-31.6 68.4-56.4 109.3-73.8 42.3-17.9 87.4-27 133.8-27 46.5 0 91.5 9.1 133.8 27a341.5 341.5 0 01109.3 73.8c9.9 9.9 19.2 20.4 27.8 31.4l-60.2 47a8 8 0 003 14.1l175.6 43c5 1.2 9.9-2.6 9.9-7.7l.8-180.9c-.1-6.6-7.8-10.3-13-6.2z"}}]},name:"reload",theme:"outlined"};var n=e.i(9583),l=r.forwardRef(function(e,l){return r.createElement(n.default,(0,t.default)({},e,{ref:l,icon:a}))});e.s(["ReloadOutlined",0,l],91979)},625901,e=>{"use strict";var t=e.i(266027),r=e.i(621482),a=e.i(243652),n=e.i(764205),l=e.i(135214);let s=(0,a.createQueryKeys)("models"),i=(0,a.createQueryKeys)("modelHub"),o=(0,a.createQueryKeys)("allProxyModels");(0,a.createQueryKeys)("selectedTeamModels");let u=(0,a.createQueryKeys)("infiniteModels");e.s(["useAllProxyModels",0,()=>{let{accessToken:e,userId:r,userRole:a}=(0,l.default)();return(0,t.useQuery)({queryKey:o.list({}),queryFn:async()=>await (0,n.modelAvailableCall)(e,r,a,!0,null,!0,!1,"expand"),enabled:!!(e&&r&&a)})},"useInfiniteModelInfo",0,(e=50,t)=>{let{accessToken:a,userId:s,userRole:i}=(0,l.default)();return(0,r.useInfiniteQuery)({queryKey:u.list({filters:{...s&&{userId:s},...i&&{userRole:i},size:e,...t&&{search:t}}}),queryFn:async({pageParam:r})=>await (0,n.modelInfoCall)(a,s,i,r,e,t),initialPageParam:1,getNextPageParam:e=>{if(e.current_page{let{accessToken:e}=(0,l.default)();return(0,t.useQuery)({queryKey:i.list({}),queryFn:async()=>await (0,n.modelHubCall)(e),enabled:!!e})},"useModelsInfo",0,(e=1,r=50,a,i,o,u,d)=>{let{accessToken:c,userId:m,userRole:h}=(0,l.default)();return(0,t.useQuery)({queryKey:s.list({filters:{...m&&{userId:m},...h&&{userRole:h},page:e,size:r,...a&&{search:a},...i&&{modelId:i},...o&&{teamId:o},...u&&{sortBy:u},...d&&{sortOrder:d}}}),queryFn:async()=>await (0,n.modelInfoCall)(c,m,h,e,r,a,i,o,u,d),enabled:!!(c&&m&&h)})}])},969550,e=>{"use strict";var t=e.i(843476),r=e.i(271645);let a=r.forwardRef(function(e,t){return r.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:t},e),r.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M3 4a1 1 0 011-1h16a1 1 0 011 1v2.586a1 1 0 01-.293.707l-6.414 6.414a1 1 0 00-.293.707V17l-4 4v-6.586a1 1 0 00-.293-.707L3.293 7.293A1 1 0 013 6.586V4z"}))});var n=e.i(464571),l=e.i(311451),s=e.i(199133),i=e.i(374009);e.s(["default",0,({options:e,onApplyFilters:o,onResetFilters:u,initialValues:d={},buttonLabel:c="Filters"})=>{let[m,h]=(0,r.useState)(!1),[f,p]=(0,r.useState)(d),[g,v]=(0,r.useState)({}),[b,x]=(0,r.useState)({}),[w,y]=(0,r.useState)({}),[C,k]=(0,r.useState)({}),M=(0,r.useCallback)((0,i.default)(async(e,t)=>{if(t.isSearchable&&t.searchFn){x(e=>({...e,[t.name]:!0}));try{let r=await t.searchFn(e);v(e=>({...e,[t.name]:r}))}catch(e){console.error("Error searching:",e),v(e=>({...e,[t.name]:[]}))}finally{x(e=>({...e,[t.name]:!1}))}}},300),[]),S=(0,r.useCallback)(async e=>{if(e.isSearchable&&e.searchFn&&!C[e.name]){x(t=>({...t,[e.name]:!0})),k(t=>({...t,[e.name]:!0}));try{let t=await e.searchFn("");v(r=>({...r,[e.name]:t}))}catch(t){console.error("Error loading initial options:",t),v(t=>({...t,[e.name]:[]}))}finally{x(t=>({...t,[e.name]:!1}))}}},[C]);(0,r.useEffect)(()=>{m&&e.forEach(e=>{e.isSearchable&&!C[e.name]&&S(e)})},[m,e,S,C]);let j=(e,t)=>{let r={...f,[e]:t};p(r),o(r)};return(0,t.jsxs)("div",{className:"w-full",children:[(0,t.jsxs)("div",{className:"flex items-center gap-2 mb-6",children:[(0,t.jsx)(n.Button,{icon:(0,t.jsx)(a,{className:"h-4 w-4"}),onClick:()=>h(!m),className:"flex items-center gap-2",children:c}),(0,t.jsx)(n.Button,{onClick:()=>{let t={};e.forEach(e=>{t[e.name]=""}),p(t),u()},children:"Reset Filters"})]}),m&&(0,t.jsx)("div",{className:"grid grid-cols-3 gap-x-6 gap-y-4 mb-6",children:["Team ID","Status","Organization ID","Key Alias","User ID","End User","Error Code","Error Message","Key Hash","Model","Public model / search tool"].map(r=>{let a,n=e.find(e=>e.label===r||e.name===r);return n?(0,t.jsxs)("div",{className:"flex flex-col gap-2",children:[(0,t.jsx)("label",{className:"text-sm text-gray-600",children:n.label||n.name}),n.isSearchable?(0,t.jsx)(s.Select,{showSearch:!0,className:"w-full",placeholder:`Search ${n.label||n.name}...`,value:f[n.name]||void 0,onChange:e=>j(n.name,e),onOpenChange:e=>{e&&n.isSearchable&&!C[n.name]&&S(n)},onSearch:e=>{y(t=>({...t,[n.name]:e})),n.searchFn&&M(e,n)},filterOption:!1,loading:b[n.name],options:g[n.name]||[],allowClear:!0,notFoundContent:b[n.name]?"Loading...":"No results found"}):n.options?(0,t.jsx)(s.Select,{className:"w-full",placeholder:`Select ${n.label||n.name}...`,value:f[n.name]||void 0,onChange:e=>j(n.name,e),allowClear:!0,children:n.options.map(e=>(0,t.jsx)(s.Select.Option,{value:e.value,children:e.label},e.value))}):n.customComponent?(a=n.customComponent,(0,t.jsx)(a,{value:f[n.name]||void 0,onChange:e=>j(n.name,e??""),placeholder:`Select ${n.label||n.name}...`,allFilters:f})):(0,t.jsx)(l.Input,{className:"w-full",placeholder:`Enter ${n.label||n.name}...`,value:f[n.name]||"",onChange:e=>j(n.name,e.target.value),allowClear:!0})]},n.name):null})})]})}],969550)},633627,e=>{"use strict";var t=e.i(764205);let r=(e,t,r,a)=>{for(let n of e){let e=n?.key_alias;e&&"string"==typeof e&&t.add(e.trim());let l=n?.organization_id??n?.org_id;l&&"string"==typeof l&&r.add(l.trim());let s=n?.user_id;if(s&&"string"==typeof s){let e=n?.user?.user_email||s;a.set(s,e)}}},a=async(e,a)=>{if(!e||!a)return{keyAliases:[],organizationIds:[],userIds:[]};try{let n=new Set,l=new Set,s=new Map,i=await (0,t.keyListCall)(e,null,a,null,null,null,1,100,null,null,"user",null),o=i?.keys||[],u=i?.total_pages??1;r(o,n,l,s);let d=Math.min(u,10)-1;if(d>0){let i=Array.from({length:d},(r,n)=>(0,t.keyListCall)(e,null,a,null,null,null,n+2,100,null,null,"user",null));for(let e of(await Promise.allSettled(i)))"fulfilled"===e.status&&r(e.value?.keys||[],n,l,s)}return{keyAliases:Array.from(n).sort(),organizationIds:Array.from(l).sort(),userIds:Array.from(s.entries()).map(([e,t])=>({id:e,email:t}))}}catch(e){return console.error("Error fetching team filter options:",e),{keyAliases:[],organizationIds:[],userIds:[]}}},n=async(e,r)=>{if(!e)return[];try{let a=[],n=1,l=!0;for(;l;){let s=await (0,t.teamListCall)(e,r||null,null);a=[...a,...s],n{if(!e)return[];try{let r=[],a=1,n=!0;for(;n;){let l=await (0,t.organizationListCall)(e);r=[...r,...l],a{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3"}))});e.s(["ClipboardCopyIcon",0,r],551332)},434626,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14"}))});e.s(["ExternalLinkIcon",0,r],434626)},902555,e=>{"use strict";var t=e.i(843476),r=e.i(591935),a=e.i(122577),n=e.i(278587),l=e.i(68155),s=e.i(360820),i=e.i(871943),o=e.i(434626),u=e.i(551332),d=e.i(592968),c=e.i(115504),m=e.i(752978);function h({icon:e,onClick:r,className:a,disabled:n,dataTestId:l}){return n?(0,t.jsx)(m.Icon,{icon:e,size:"sm",className:"opacity-50 cursor-not-allowed","data-testid":l}):(0,t.jsx)(m.Icon,{icon:e,size:"sm",onClick:r,className:(0,c.cx)("cursor-pointer",a),"data-testid":l})}let f={Edit:{icon:r.PencilAltIcon,className:"hover:text-blue-600"},Delete:{icon:l.TrashIcon,className:"hover:text-red-600"},Test:{icon:a.PlayIcon,className:"hover:text-blue-600"},Regenerate:{icon:n.RefreshIcon,className:"hover:text-green-600"},Up:{icon:s.ChevronUpIcon,className:"hover:text-blue-600"},Down:{icon:i.ChevronDownIcon,className:"hover:text-blue-600"},Open:{icon:o.ExternalLinkIcon,className:"hover:text-green-600"},Copy:{icon:u.ClipboardCopyIcon,className:"hover:text-blue-600"}};function p({onClick:e,tooltipText:r,disabled:a=!1,disabledTooltipText:n,dataTestId:l,variant:s}){let{icon:i,className:o}=f[s];return(0,t.jsx)(d.Tooltip,{title:a?n:r,children:(0,t.jsx)("span",{children:(0,t.jsx)(h,{icon:i,onClick:e,className:o,disabled:a,dataTestId:l})})})}e.s(["default",()=>p],902555)},122577,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M14.752 11.168l-3.197-2.132A1 1 0 0010 9.87v4.263a1 1 0 001.555.832l3.197-2.132a1 1 0 000-1.664z"}),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M21 12a9 9 0 11-18 0 9 9 0 0118 0z"}))});e.s(["PlayIcon",0,r],122577)},68155,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"}))});e.s(["TrashIcon",0,r],68155)},871943,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19 9l-7 7-7-7"}))});e.s(["ChevronDownIcon",0,r],871943)},360820,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M5 15l7-7 7 7"}))});e.s(["ChevronUpIcon",0,r],360820)},278587,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"}))});e.s(["RefreshIcon",0,r],278587)},207670,e=>{"use strict";function t(){for(var e,t,r=0,a="",n=arguments.length;rt,"default",0,t])},728889,e=>{"use strict";var t=e.i(290571),r=e.i(271645),a=e.i(829087),n=e.i(480731),l=e.i(444755),s=e.i(673706),i=e.i(95779);let o={xs:{paddingX:"px-1.5",paddingY:"py-1.5"},sm:{paddingX:"px-1.5",paddingY:"py-1.5"},md:{paddingX:"px-2",paddingY:"py-2"},lg:{paddingX:"px-2",paddingY:"py-2"},xl:{paddingX:"px-2.5",paddingY:"py-2.5"}},u={xs:{height:"h-3",width:"w-3"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-7",width:"w-7"},xl:{height:"h-9",width:"w-9"}},d={simple:{rounded:"",border:"",ring:"",shadow:""},light:{rounded:"rounded-tremor-default",border:"",ring:"",shadow:""},shadow:{rounded:"rounded-tremor-default",border:"border",ring:"",shadow:"shadow-tremor-card dark:shadow-dark-tremor-card"},solid:{rounded:"rounded-tremor-default",border:"border-2",ring:"ring-1",shadow:""},outlined:{rounded:"rounded-tremor-default",border:"border",ring:"ring-2",shadow:""}},c=(0,s.makeClassName)("Icon"),m=r.default.forwardRef((e,m)=>{let{icon:h,variant:f="simple",tooltip:p,size:g=n.Sizes.SM,color:v,className:b}=e,x=(0,t.__rest)(e,["icon","variant","tooltip","size","color","className"]),w=((e,t)=>{switch(e){case"simple":return{textColor:t?(0,s.getColorClassNames)(t,i.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:"",borderColor:"",ringColor:""};case"light":return{textColor:t?(0,s.getColorClassNames)(t,i.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.tremorTwMerge)((0,s.getColorClassNames)(t,i.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand-muted dark:bg-dark-tremor-brand-muted",borderColor:"",ringColor:""};case"shadow":return{textColor:t?(0,s.getColorClassNames)(t,i.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.tremorTwMerge)((0,s.getColorClassNames)(t,i.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:"border-tremor-border dark:border-dark-tremor-border",ringColor:""};case"solid":return{textColor:t?(0,s.getColorClassNames)(t,i.colorPalette.text).textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:t?(0,l.tremorTwMerge)((0,s.getColorClassNames)(t,i.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-brand dark:bg-dark-tremor-brand",borderColor:"border-tremor-brand-inverted dark:border-dark-tremor-brand-inverted",ringColor:"ring-tremor-ring dark:ring-dark-tremor-ring"};case"outlined":return{textColor:t?(0,s.getColorClassNames)(t,i.colorPalette.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.tremorTwMerge)((0,s.getColorClassNames)(t,i.colorPalette.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:t?(0,s.getColorClassNames)(t,i.colorPalette.ring).borderColor:"border-tremor-brand-subtle dark:border-dark-tremor-brand-subtle",ringColor:t?(0,l.tremorTwMerge)((0,s.getColorClassNames)(t,i.colorPalette.ring).ringColor,"ring-opacity-40"):"ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted"}}})(f,v),{tooltipProps:y,getReferenceProps:C}=(0,a.useTooltip)();return r.default.createElement("span",Object.assign({ref:(0,s.mergeRefs)([m,y.refs.setReference]),className:(0,l.tremorTwMerge)(c("root"),"inline-flex shrink-0 items-center justify-center",w.bgColor,w.textColor,w.borderColor,w.ringColor,d[f].rounded,d[f].border,d[f].shadow,d[f].ring,o[g].paddingX,o[g].paddingY,b)},C,x),r.default.createElement(a.default,Object.assign({text:p},y)),r.default.createElement(h,{className:(0,l.tremorTwMerge)(c("icon"),"shrink-0",u[g].height,u[g].width)}))});m.displayName="Icon",e.s(["default",()=>m],728889)},752978,e=>{"use strict";var t=e.i(728889);e.s(["Icon",()=>t.default])},591935,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"}))});e.s(["PencilAltIcon",0,r],591935)},907308,e=>{"use strict";var t=e.i(843476),r=e.i(271645),a=e.i(212931),n=e.i(808613),l=e.i(464571),s=e.i(199133),i=e.i(592968),o=e.i(213205),u=e.i(374009),d=e.i(764205);e.s(["default",0,({isVisible:e,onCancel:c,onSubmit:m,accessToken:h,title:f="Add Team Member",roles:p=[{label:"admin",value:"admin",description:"Admin role. Can create team keys, add members, and manage settings."},{label:"user",value:"user",description:"User role. Can view team info, but not manage it."}],defaultRole:g="user",teamId:v})=>{let[b]=n.Form.useForm(),[x,w]=(0,r.useState)([]),[y,C]=(0,r.useState)(!1),[k,M]=(0,r.useState)("user_email"),[S,j]=(0,r.useState)(!1),E=async(e,t)=>{if(!e)return void w([]);C(!0);try{let r=new URLSearchParams;if(r.append(t,e),v&&r.append("team_id",v),null==h)return;let a=(await (0,d.userFilterUICall)(h,r)).map(e=>({label:"user_email"===t?`${e.user_email}`:`${e.user_id}`,value:"user_email"===t?e.user_email:e.user_id,user:e}));w(a)}catch(e){console.error("Error fetching users:",e)}finally{C(!1)}},N=(0,r.useCallback)((0,u.default)((e,t)=>E(e,t),300),[]),T=(e,t)=>{M(t),N(e,t)},$=(e,t)=>{let r=t.user;b.setFieldsValue({user_email:r.user_email,user_id:r.user_id,role:b.getFieldValue("role")})},O=async e=>{j(!0);try{await m(e)}finally{j(!1)}};return(0,t.jsx)(a.Modal,{title:f,open:e,onCancel:()=>{b.resetFields(),w([]),c()},footer:null,width:800,maskClosable:!S,children:(0,t.jsxs)(n.Form,{form:b,onFinish:O,labelCol:{span:8},wrapperCol:{span:16},labelAlign:"left",initialValues:{role:g},children:[(0,t.jsx)(n.Form.Item,{label:"Email",name:"user_email",className:"mb-4",children:(0,t.jsx)(s.Select,{showSearch:!0,className:"w-full",placeholder:"Search by email",filterOption:!1,onSearch:e=>T(e,"user_email"),onSelect:(e,t)=>$(e,t),options:"user_email"===k?x:[],loading:y,allowClear:!0,"data-testid":"member-email-search"})}),(0,t.jsx)("div",{className:"text-center mb-4",children:"OR"}),(0,t.jsx)(n.Form.Item,{label:"User ID",name:"user_id",className:"mb-4",children:(0,t.jsx)(s.Select,{showSearch:!0,className:"w-full",placeholder:"Search by user ID",filterOption:!1,onSearch:e=>T(e,"user_id"),onSelect:(e,t)=>$(e,t),options:"user_id"===k?x:[],loading:y,allowClear:!0})}),(0,t.jsx)(n.Form.Item,{label:"Member Role",name:"role",className:"mb-4",children:(0,t.jsx)(s.Select,{defaultValue:g,children:p.map(e=>(0,t.jsx)(s.Select.Option,{value:e.value,children:(0,t.jsxs)(i.Tooltip,{title:e.description,children:[(0,t.jsx)("span",{className:"font-medium",children:e.label}),(0,t.jsxs)("span",{className:"ml-2 text-gray-500 text-sm",children:["- ",e.description]})]})},e.value))})}),(0,t.jsx)("div",{className:"text-right mt-4",children:(0,t.jsx)(l.Button,{type:"primary",htmlType:"submit",icon:(0,t.jsx)(o.UserAddOutlined,{}),loading:S,children:S?"Adding...":"Add Member"})})]})})}])},162386,e=>{"use strict";var t=e.i(843476),r=e.i(625901),a=e.i(109799),n=e.i(785242),l=e.i(738014),s=e.i(199133),i=e.i(981339),o=e.i(592968);let u={label:"All Proxy Models",value:"all-proxy-models"},d={label:"No Default Models",value:"no-default-models"},c=[u,d],m={user:({allProxyModels:e,userModels:t,options:r})=>t&&r?.includeUserModels?t:[],team:({allProxyModels:e,selectedOrganization:t,userModels:r})=>t?t.models.includes(u.value)||0===t.models.length?e:e.filter(e=>t.models.includes(e)):e??[],organization:({allProxyModels:e})=>e,global:({allProxyModels:e})=>e};e.s(["ModelSelect",0,e=>{let{teamID:h,organizationID:f,options:p,context:g,dataTestId:v,value:b=[],onChange:x,style:w}=e,{includeUserModels:y,showAllTeamModelsOption:C,showAllProxyModelsOverride:k,includeSpecialOptions:M}=p||{},{data:S,isLoading:j}=(0,r.useAllProxyModels)(),{data:E,isLoading:N}=(0,n.useTeam)(h),{data:T,isLoading:$}=(0,a.useOrganization)(f),{data:O,isLoading:_}=(0,l.useCurrentUser)(),I=e=>c.some(t=>t.value===e),L=b.some(I),A=T?.models.includes(u.value)||T?.models.length===0;if(j||N||$||_)return(0,t.jsx)(i.Skeleton.Input,{active:!0,block:!0});let{wildcard:D,regular:R}=(e=>{let t=[],r=[];for(let a of e)a.endsWith("/*")?t.push(a):r.push(a);return{wildcard:t,regular:r}})(((e,t,r)=>{let a=Array.from(new Map(e.map(e=>[e.id,e])).values()).map(e=>e.id);if(t.options?.showAllProxyModelsOverride)return a;let n=m[t.context];return n?n({allProxyModels:a,...r,options:t.options}):[]})(S?.data??[],e,{selectedTeam:E,selectedOrganization:T,userModels:O?.models}));return(0,t.jsx)(s.Select,{"data-testid":v,value:b,onChange:e=>{let t=e.filter(I);x(t.length>0?[t[t.length-1]]:e)},style:w,options:[...M?[{label:(0,t.jsx)("span",{children:"Special Options"}),title:"Special Options",options:[...k||A&&M||"global"===g?[{label:(0,t.jsx)("span",{children:"All Proxy Models"}),value:u.value,disabled:b.length>0&&b.some(e=>I(e)&&e!==u.value),key:u.value}]:[],{label:(0,t.jsx)("span",{children:"No Default Models"}),value:d.value,disabled:b.length>0&&b.some(e=>I(e)&&e!==d.value),key:d.value}]}]:[],...D.length>0?[{label:(0,t.jsx)("span",{children:"Wildcard Options"}),title:"Wildcard Options",options:D.map(e=>{let r=e.replace("/*",""),a=r.charAt(0).toUpperCase()+r.slice(1);return{label:(0,t.jsx)("span",{children:`All ${a} models`}),value:e,disabled:L}})}]:[],{label:(0,t.jsx)("span",{children:"Models"}),title:"Models",options:R.map(e=>({label:(0,t.jsx)("span",{children:e}),value:e,disabled:L}))}],mode:"multiple",placeholder:"Select Models",allowClear:!0,maxTagCount:"responsive",maxTagPlaceholder:e=>(0,t.jsx)(o.Tooltip,{styles:{root:{pointerEvents:"none"}},title:e.map(({value:e})=>e).join(", "),children:(0,t.jsxs)("span",{children:["+",e.length," more"]})})})}],162386)},276173,e=>{"use strict";var t=e.i(843476),r=e.i(599724),a=e.i(779241),n=e.i(464571),l=e.i(808613),s=e.i(212931),i=e.i(199133),o=e.i(271645),u=e.i(435451);e.s(["default",0,({visible:e,onCancel:d,onSubmit:c,initialData:m,mode:h,config:f})=>{let p,[g]=l.Form.useForm(),[v,b]=(0,o.useState)(!1);console.log("Initial Data:",m),(0,o.useEffect)(()=>{if(e)if("edit"===h&&m){let e={...m,role:m.role||f.defaultRole,max_budget_in_team:m.max_budget_in_team||null,tpm_limit:m.tpm_limit||null,rpm_limit:m.rpm_limit||null,allowed_models:m.allowed_models||[]};console.log("Setting form values:",e),g.setFieldsValue(e)}else g.resetFields(),g.setFieldsValue({role:f.defaultRole||f.roleOptions[0]?.value})},[e,m,h,g,f.defaultRole,f.roleOptions]);let x=async e=>{try{b(!0);let t=Object.entries(e).reduce((e,[t,r])=>{if("string"==typeof r){let a=r.trim();return""===a&&("max_budget_in_team"===t||"tpm_limit"===t||"rpm_limit"===t)?{...e,[t]:null}:{...e,[t]:a}}return{...e,[t]:r}},{});console.log("Submitting form data:",t),await Promise.resolve(c(t)),g.resetFields()}catch(e){console.error("Form submission error:",e)}finally{b(!1)}};return(0,t.jsx)(s.Modal,{title:f.title||("add"===h?"Add Member":"Edit Member"),open:e,width:1e3,footer:null,onCancel:d,children:(0,t.jsxs)(l.Form,{form:g,onFinish:x,labelCol:{span:8},wrapperCol:{span:16},labelAlign:"left",children:[f.showEmail&&(0,t.jsx)(l.Form.Item,{label:"Email",name:"user_email",className:"mb-4",rules:[{type:"email",message:"Please enter a valid email!"}],children:(0,t.jsx)(a.TextInput,{placeholder:"user@example.com"})}),f.showEmail&&f.showUserId&&(0,t.jsx)("div",{className:"text-center mb-4",children:(0,t.jsx)(r.Text,{children:"OR"})}),f.showUserId&&(0,t.jsx)(l.Form.Item,{label:"User ID",name:"user_id",className:"mb-4",children:(0,t.jsx)(a.TextInput,{placeholder:"user_123"})}),(0,t.jsx)(l.Form.Item,{label:(0,t.jsxs)("div",{className:"flex items-center gap-2",children:[(0,t.jsx)("span",{children:"Role"}),"edit"===h&&m&&(0,t.jsxs)("span",{className:"text-gray-500 text-sm",children:["(Current: ",(p=m.role,f.roleOptions.find(e=>e.value===p)?.label||p),")"]})]}),name:"role",className:"mb-4",rules:[{required:!0,message:"Please select a role!"}],children:(0,t.jsx)(i.Select,{children:"edit"===h&&m?[...f.roleOptions.filter(e=>e.value===m.role),...f.roleOptions.filter(e=>e.value!==m.role)].map(e=>(0,t.jsx)(i.Select.Option,{value:e.value,children:e.label},e.value)):f.roleOptions.map(e=>(0,t.jsx)(i.Select.Option,{value:e.value,children:e.label},e.value))})}),f.additionalFields?.map(e=>(0,t.jsx)(l.Form.Item,{label:e.label,name:e.name,className:"mb-4",rules:e.rules,children:(e=>{switch(e.type){case"input":return(0,t.jsx)(a.TextInput,{placeholder:e.placeholder});case"numerical":return(0,t.jsx)(u.default,{step:e.step||1,min:e.min||0,style:{width:"100%"},placeholder:e.placeholder||"Enter a numerical value"});case"select":return(0,t.jsx)(i.Select,{children:e.options?.map(e=>(0,t.jsx)(i.Select.Option,{value:e.value,children:e.label},e.value))});case"multi-select":return(0,t.jsx)(i.Select,{mode:"multiple",placeholder:e.placeholder||"Select options",options:e.options,allowClear:!0});default:return null}})(e)},e.name)),(0,t.jsxs)("div",{className:"text-right mt-6",children:[(0,t.jsx)(n.Button,{onClick:d,className:"mr-2",disabled:v,children:"Cancel"}),(0,t.jsx)(n.Button,{type:"default",htmlType:"submit",loading:v,children:"add"===h?v?"Adding...":"Add Member":v?"Saving...":"Save Changes"})]})]})})}])},294612,e=>{"use strict";var t=e.i(843476),r=e.i(100486),a=e.i(827252),n=e.i(213205),l=e.i(771674),s=e.i(464571),i=e.i(770914),o=e.i(291542),u=e.i(262218),d=e.i(592968),c=e.i(898586),m=e.i(902555);let{Text:h}=c.Typography;function f({members:e,canEdit:c,onEdit:f,onDelete:p,onAddMember:g,roleColumnTitle:v="Role",roleTooltip:b,extraColumns:x=[],showDeleteForMember:w,emptyText:y}){let C=[{title:"User Email",dataIndex:"user_email",key:"user_email",render:e=>(0,t.jsx)(h,{children:e||"-"})},{title:"User ID",dataIndex:"user_id",key:"user_id",render:e=>"default_user_id"===e?(0,t.jsx)(u.Tag,{color:"blue",children:"Default Proxy Admin"}):(0,t.jsx)(h,{children:e||"-"})},{title:b?(0,t.jsxs)(i.Space,{direction:"horizontal",children:[v,(0,t.jsx)(d.Tooltip,{title:b,children:(0,t.jsx)(a.InfoCircleOutlined,{})})]}):v,dataIndex:"role",key:"role",render:e=>(0,t.jsxs)(i.Space,{children:[e?.toLowerCase()==="admin"||e?.toLowerCase()==="org_admin"?(0,t.jsx)(r.CrownOutlined,{}):(0,t.jsx)(l.UserOutlined,{}),(0,t.jsx)(h,{style:{textTransform:"capitalize"},children:e||"-"})]})},...x,{title:"Actions",key:"actions",fixed:"right",width:120,render:(e,r)=>c?(0,t.jsxs)(i.Space,{children:[(0,t.jsx)(m.default,{variant:"Edit",tooltipText:"Edit member",dataTestId:"edit-member",onClick:()=>f(r)}),(!w||w(r))&&(0,t.jsx)(m.default,{variant:"Delete",tooltipText:"Delete member",dataTestId:"delete-member",onClick:()=>p(r)})]}):null}];return(0,t.jsxs)(i.Space,{direction:"vertical",style:{width:"100%"},children:[(0,t.jsxs)("span",{className:"inline-flex text-sm text-gray-700",children:[e.length," Member",1!==e.length?"s":""]}),(0,t.jsx)(o.Table,{columns:C,dataSource:e,rowKey:e=>e.user_id??e.user_email??JSON.stringify(e),pagination:!1,size:"small",scroll:{x:"max-content"},locale:y?{emptyText:y}:void 0}),g&&c&&(0,t.jsx)(s.Button,{icon:(0,t.jsx)(n.UserAddOutlined,{}),type:"primary",onClick:g,children:"Add Member"})]})}e.s(["default",()=>f])}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/06b2ea8c776c2e9b.js b/litellm/proxy/_experimental/out/_next/static/chunks/06b2ea8c776c2e9b.js deleted file mode 100644 index fe571604a7f7..000000000000 --- a/litellm/proxy/_experimental/out/_next/static/chunks/06b2ea8c776c2e9b.js +++ /dev/null @@ -1 +0,0 @@ -(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,233538,e=>{"use strict";function t(e){let t=e.parentElement,r=null;for(;t&&!(t instanceof HTMLFieldSetElement);)t instanceof HTMLLegendElement&&(r=t),t=t.parentElement;let l=(null==t?void 0:t.getAttribute("disabled"))==="";return!(l&&function(e){if(!e)return!1;let t=e.previousElementSibling;for(;null!==t;){if(t instanceof HTMLLegendElement)return!1;t=t.previousElementSibling}return!0}(r))&&l}e.s(["isDisabledReactIssue7711",()=>t])},220508,e=>{"use strict";var t=e.i(271645);let r=t.forwardRef(function(e,r){return t.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",strokeWidth:2,stroke:"currentColor","aria-hidden":"true",ref:r},e),t.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"}))});e.s(["CheckCircleIcon",0,r],220508)},503269,214520,601893,694421,140721,942803,35889,722678,e=>{"use strict";var t=e.i(271645),r=e.i(914189);function l(e,l,s){let[a,n]=(0,t.useState)(s),i=void 0!==e,o=(0,t.useRef)(i),c=(0,t.useRef)(!1),d=(0,t.useRef)(!1);return!i||o.current||c.current?i||!o.current||d.current||(d.current=!0,o.current=i,console.error("A component is changing from controlled to uncontrolled. This may be caused by the value changing from a defined value to undefined, which should not happen.")):(c.current=!0,o.current=i,console.error("A component is changing from uncontrolled to controlled. This may be caused by the value changing from undefined to a defined value, which should not happen.")),[i?e:a,(0,r.useEvent)(e=>(i||n(e),null==l?void 0:l(e)))]}function s(e){let[r]=(0,t.useState)(e);return r}e.s(["useControllable",()=>l],503269),e.s(["useDefaultValue",()=>s],214520);let a=(0,t.createContext)(void 0);function n(){return(0,t.useContext)(a)}e.s(["useDisabled",()=>n],601893);var i=e.i(174080),o=e.i(746725);function c(e={},t=null,r=[]){for(let[l,s]of Object.entries(e))!function e(t,r,l){if(Array.isArray(l))for(let[s,a]of l.entries())e(t,d(r,s.toString()),a);else l instanceof Date?t.push([r,l.toISOString()]):"boolean"==typeof l?t.push([r,l?"1":"0"]):"string"==typeof l?t.push([r,l]):"number"==typeof l?t.push([r,`${l}`]):null==l?t.push([r,""]):c(l,r,t)}(r,d(t,l),s);return r}function d(e,t){return e?e+"["+t+"]":t}function u(e){var t,r;let l=null!=(t=null==e?void 0:e.form)?t:e.closest("form");if(l){for(let t of l.elements)if(t!==e&&("INPUT"===t.tagName&&"submit"===t.type||"BUTTON"===t.tagName&&"submit"===t.type||"INPUT"===t.nodeName&&"image"===t.type))return void t.click();null==(r=l.requestSubmit)||r.call(l)}}e.s(["attemptSubmit",()=>u,"objectToFormEntries",()=>c],694421);var m=e.i(700020),f=e.i(2788);let h=(0,t.createContext)(null);function g({children:e}){let r=(0,t.useContext)(h);if(!r)return t.default.createElement(t.default.Fragment,null,e);let{target:l}=r;return l?(0,i.createPortal)(t.default.createElement(t.default.Fragment,null,e),l):null}function p({data:e,form:r,disabled:l,onReset:s,overrides:a}){let[n,i]=(0,t.useState)(null),d=(0,o.useDisposables)();return(0,t.useEffect)(()=>{if(s&&n)return d.addEventListener(n,"reset",s)},[n,r,s]),t.default.createElement(g,null,t.default.createElement(x,{setForm:i,formId:r}),c(e).map(([e,s])=>t.default.createElement(f.Hidden,{features:f.HiddenFeatures.Hidden,...(0,m.compact)({key:e,as:"input",type:"hidden",hidden:!0,readOnly:!0,form:r,disabled:l,name:e,value:s,...a})})))}function x({setForm:e,formId:r}){return(0,t.useEffect)(()=>{if(r){let t=document.getElementById(r);t&&e(t)}},[e,r]),r?null:t.default.createElement(f.Hidden,{features:f.HiddenFeatures.Hidden,as:"input",type:"hidden",hidden:!0,readOnly:!0,ref:t=>{if(!t)return;let r=t.closest("form");r&&e(r)}})}e.s(["FormFields",()=>p],140721);let b=(0,t.createContext)(void 0);function y(){return(0,t.useContext)(b)}e.s(["useProvidedId",()=>y],942803);var v=e.i(835696),j=e.i(294316);let k=(0,t.createContext)(null);function N(){var e,r;return null!=(r=null==(e=(0,t.useContext)(k))?void 0:e.value)?r:void 0}function w(){let[e,l]=(0,t.useState)([]);return[e.length>0?e.join(" "):void 0,(0,t.useMemo)(()=>function(e){let s=(0,r.useEvent)(e=>(l(t=>[...t,e]),()=>l(t=>{let r=t.slice(),l=r.indexOf(e);return -1!==l&&r.splice(l,1),r}))),a=(0,t.useMemo)(()=>({register:s,slot:e.slot,name:e.name,props:e.props,value:e.value}),[s,e.slot,e.name,e.props,e.value]);return t.default.createElement(k.Provider,{value:a},e.children)},[l])]}k.displayName="DescriptionContext";let S=Object.assign((0,m.forwardRefWithAs)(function(e,r){let l=(0,t.useId)(),s=n(),{id:a=`headlessui-description-${l}`,...i}=e,o=function e(){let r=(0,t.useContext)(k);if(null===r){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return r}(),c=(0,j.useSyncRefs)(r);(0,v.useIsoMorphicEffect)(()=>o.register(a),[a,o.register]);let d=s||!1,u=(0,t.useMemo)(()=>({...o.slot,disabled:d}),[o.slot,d]),f={ref:c,...o.props,id:a};return(0,m.useRender)()({ourProps:f,theirProps:i,slot:u,defaultTag:"p",name:o.name||"Description"})}),{});e.s(["Description",()=>S,"useDescribedBy",()=>N,"useDescriptions",()=>w],35889);let C=(0,t.createContext)(null);function M(e){var r,l,s;let a=null!=(l=null==(r=(0,t.useContext)(C))?void 0:r.value)?l:void 0;return(null!=(s=null==e?void 0:e.length)?s:0)>0?[a,...e].filter(Boolean).join(" "):a}function O({inherit:e=!1}={}){let l=M(),[s,a]=(0,t.useState)([]),n=e?[l,...s].filter(Boolean):s;return[n.length>0?n.join(" "):void 0,(0,t.useMemo)(()=>function(e){let l=(0,r.useEvent)(e=>(a(t=>[...t,e]),()=>a(t=>{let r=t.slice(),l=r.indexOf(e);return -1!==l&&r.splice(l,1),r}))),s=(0,t.useMemo)(()=>({register:l,slot:e.slot,name:e.name,props:e.props,value:e.value}),[l,e.slot,e.name,e.props,e.value]);return t.default.createElement(C.Provider,{value:s},e.children)},[a])]}C.displayName="LabelContext";let E=Object.assign((0,m.forwardRefWithAs)(function(e,l){var s;let a=(0,t.useId)(),i=function e(){let r=(0,t.useContext)(C);if(null===r){let t=Error("You used a