From e3956bf01189ef1d6aa4db4b055aed4c4ada4b5c Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 17:43:30 +0000 Subject: [PATCH 01/15] feat(decisions): add the OpenAI Decisions spec types and the System One translation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/base_llm/decisions/systemone.py | 217 ++++++++++++++++ litellm/types/openai_decisions.py | 207 +++++++++++++++ .../unit/llms/base_llm/decisions/__init__.py | 0 .../llms/base_llm/decisions/test_systemone.py | 239 ++++++++++++++++++ tests/unit/types/test_openai_decisions.py | 105 ++++++++ 5 files changed, 768 insertions(+) create mode 100644 litellm/llms/base_llm/decisions/systemone.py create mode 100644 litellm/types/openai_decisions.py create mode 100644 tests/unit/llms/base_llm/decisions/__init__.py create mode 100644 tests/unit/llms/base_llm/decisions/test_systemone.py create mode 100644 tests/unit/types/test_openai_decisions.py diff --git a/litellm/llms/base_llm/decisions/systemone.py b/litellm/llms/base_llm/decisions/systemone.py new file mode 100644 index 000000000000..699927924005 --- /dev/null +++ b/litellm/llms/base_llm/decisions/systemone.py @@ -0,0 +1,217 @@ +"""The Jev / System One wire shape and its translation to and from the OpenAI Decisions shape. + +System One (TypeSafe, Perplexity, OpenRouter, Cloudflare Clef, Strands Decider) takes +{"model", "state", "questions": {name: question}} and answers with {"model", "answers": {name: answer}, "usage"}. +Predicates are `noul` questions, choice options are a `criteria` map, score levels are a `criteria` list. +""" + +import itertools +from collections.abc import Mapping, Sequence +from typing import Final, Literal, TypeAlias + +from pydantic import ConfigDict, TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.base import LiteLLMPydanticObjectBase +from litellm.types.openai_decisions import ( + ChoiceAnswer, + ChoiceProbability, + ChoiceQuestion, + DecisionAnswer, + DecisionChoice, + DecisionInputMessage, + DecisionInputPart, + DecisionQuestion, + DecisionsRequest, + DecisionsResponse, + DecisionsUsage, + PredicateAnswer, + PredicateQuestion, + ScoreAnswer, + ScoreProbability, + ScoreQuestion, +) + +SystemOneJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] + + +class SystemOneNoulAnswer(LiteLLMPydanticObjectBase): + type: Literal["noul"] + noul: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class SystemOneChoiceAnswer(LiteLLMPydanticObjectBase): + type: Literal["choice"] + choice: str + confidence: float + probabilities: Mapping[str, float] + + model_config = ConfigDict(extra="allow", frozen=True) + + +class SystemOneScoreAnswer(LiteLLMPydanticObjectBase): + type: Literal["score"] + score: float + confidence: float + probabilities: Mapping[str, float] + + model_config = ConfigDict(extra="allow", frozen=True) + + +SystemOneAnswer: TypeAlias = SystemOneNoulAnswer | SystemOneChoiceAnswer | SystemOneScoreAnswer + + +class SystemOneUsage(LiteLLMPydanticObjectBase): + input_tokens: int = 0 + output_tokens: int = 0 + + model_config = ConfigDict(extra="allow", frozen=True) + + +class SystemOneResponse(LiteLLMPydanticObjectBase): + model: str | None = None + answers: Mapping[str, SystemOneAnswer] + usage: SystemOneUsage | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +SYSTEM_ONE_RESPONSE_ADAPTER: Final[TypeAdapter[SystemOneResponse]] = TypeAdapter(SystemOneResponse) + + +def _unsupported(what: str, custom_llm_provider: str) -> BaseLLMException: + return BaseLLMException( + status_code=400, + message=f"Decisions provider '{custom_llm_provider}' does not support {what}", + ) + + +def question_keys(questions: Sequence[DecisionQuestion], custom_llm_provider: str) -> tuple[str, ...]: + """System One keys questions and answers by name, so unnamed questions get a positional key.""" + names: Final = tuple(question.name for question in questions if question.name is not None) + if len(set(names)) != len(names): + raise BaseLLMException( + status_code=400, + message=f"Decisions provider '{custom_llm_provider}' requires a unique name per question", + ) + taken: Final = frozenset(names) + return tuple( + question.name if question.name is not None else _positional_key(index, taken) + for index, question in enumerate(questions) + ) + + +def _positional_key(index: int, taken: frozenset[str]) -> str: + return next(key for key in (f"{'_' * depth}q{index}" for depth in itertools.count()) if key not in taken) + + +def system_one_state(request: DecisionsRequest, custom_llm_provider: str) -> str: + if isinstance(request.input, str): + return request.input + return "\n".join(_message_text(message, custom_llm_provider) for message in request.input) + + +def _message_text(message: DecisionInputMessage, custom_llm_provider: str) -> str: + if isinstance(message.content, str): + return message.content + return "\n".join(_part_text(part, custom_llm_provider) for part in message.content) + + +def _part_text(part: DecisionInputPart, custom_llm_provider: str) -> str: + if part.type != "input_text": + raise _unsupported("input_image parts", custom_llm_provider) + return part.text + + +def system_one_question(question: DecisionQuestion, custom_llm_provider: str) -> dict[str, object]: + if isinstance(question, PredicateQuestion): + return {"type": "noul", "instructions": question.instructions} + if isinstance(question, ChoiceQuestion): + values: Final = tuple(_choice_key(choice, custom_llm_provider) for choice in question.choices) + if len(set(values)) != len(values): + raise _unsupported("repeated choice values", custom_llm_provider) + criteria: Final = {value: choice.description for value, choice in zip(values, question.choices, strict=True)} + return {"type": "choice", "instructions": question.instructions, "criteria": criteria} + return { + "type": "score", + "instructions": question.instructions, + "criteria": [level.description if level.description is not None else level.label for level in question.levels], + } + + +def system_one_request(model: str, request: DecisionsRequest, custom_llm_provider: str) -> dict[str, object]: + keys: Final = question_keys(request.questions, custom_llm_provider) + return { + "model": model, + "state": system_one_state(request, custom_llm_provider), + "questions": { + key: system_one_question(question, custom_llm_provider) + for key, question in zip(keys, request.questions, strict=True) + }, + } + + +def _answer_for( + key: str, + question: DecisionQuestion, + answers: Mapping[str, SystemOneAnswer], + custom_llm_provider: str, +) -> DecisionAnswer: + answer: Final = answers.get(key) + if isinstance(question, PredicateQuestion) and isinstance(answer, SystemOneNoulAnswer): + return PredicateAnswer(type="predicate", name=question.name, probability=answer.noul) + if isinstance(question, ChoiceQuestion) and isinstance(answer, SystemOneChoiceAnswer): + return ChoiceAnswer( + type="choice", + name=question.name, + choice=answer.choice, + probabilities=[ + ChoiceProbability(value=choice.value, probability=answer.probabilities.get(str(choice.value), 0.0)) + for choice in question.choices + ], + confidence=answer.confidence, + ) + if isinstance(question, ScoreQuestion) and isinstance(answer, SystemOneScoreAnswer): + return ScoreAnswer( + type="score", + name=question.name, + score=answer.score, + probabilities=[ + ScoreProbability(value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0)) + for index, level in enumerate(question.levels) + ], + confidence=answer.confidence, + ) + raise BaseLLMException( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned no {question.type} answer for question '{key}'", + ) + + +def decisions_response( + system_one: SystemOneResponse, + request: DecisionsRequest, + custom_llm_provider: str, +) -> DecisionsResponse: + keys: Final = question_keys(request.questions, custom_llm_provider) + usage: Final = system_one.usage if system_one.usage is not None else SystemOneUsage() + return DecisionsResponse( + model=system_one.model, + answers=[ + _answer_for(key, question, system_one.answers, custom_llm_provider) + for key, question in zip(keys, request.questions, strict=True) + ], + usage=DecisionsUsage( + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ), + ) + + +def _choice_key(choice: DecisionChoice, custom_llm_provider: str) -> str: + if not isinstance(choice.value, str): + raise _unsupported("boolean choice values", custom_llm_provider) + return choice.value diff --git a/litellm/types/openai_decisions.py b/litellm/types/openai_decisions.py new file mode 100644 index 000000000000..6fcbe9ff61e8 --- /dev/null +++ b/litellm/types/openai_decisions.py @@ -0,0 +1,207 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field, PrivateAttr, with_config +from typing_extensions import ReadOnly, Required, TypedDict + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +ChoiceValue: TypeAlias = str | bool + + +class DecisionInputText(LiteLLMPydanticObjectBase): + type: Literal["input_text"] + text: str + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionInputImage(LiteLLMPydanticObjectBase): + type: Literal["input_image"] + image_url: str + detail: Literal["low", "high", "auto", "original"] | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionInputPart: TypeAlias = Annotated[DecisionInputText | DecisionInputImage, Field(discriminator="type")] + + +class DecisionInputMessage(LiteLLMPydanticObjectBase): + role: Literal["user"] + content: str | Sequence[DecisionInputPart] + type: Literal["message"] | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionsInput: TypeAlias = str | Sequence[DecisionInputMessage] + + +class DecisionChoice(LiteLLMPydanticObjectBase): + value: ChoiceValue + description: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionLevel(LiteLLMPydanticObjectBase): + label: str + description: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +class PredicateQuestion(LiteLLMPydanticObjectBase): + type: Literal["predicate"] + instructions: str + name: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: str + choices: Annotated[Sequence[DecisionChoice], Field(min_length=1)] + name: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: str + levels: Annotated[Sequence[DecisionLevel], Field(min_length=1)] + name: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionQuestion: TypeAlias = Annotated[ + PredicateQuestion | ChoiceQuestion | ScoreQuestion, + Field(discriminator="type"), +] + +DecisionQuestions: TypeAlias = Annotated[Sequence[DecisionQuestion], Field(min_length=1)] + + +class DecisionsRequestBody(LiteLLMPydanticObjectBase): + input: DecisionsInput + questions: DecisionQuestions + safety_identifier: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsRequest(DecisionsRequestBody): + model: str + + +@with_config(ConfigDict(extra="allow")) +class DecisionsCallParams(TypedDict, total=False): + model: Required[ReadOnly[str]] + input: Required[ReadOnly[DecisionsInput]] + questions: Required[ReadOnly[DecisionQuestions]] + safety_identifier: ReadOnly[str | None] + api_key: ReadOnly[str | None] + api_base: ReadOnly[str | None] + timeout: ReadOnly[float | None] + custom_llm_provider: ReadOnly[str | None] + extra_headers: ReadOnly[Mapping[str, str] | None] + + +class PredicateAnswer(LiteLLMPydanticObjectBase): + type: Literal["predicate"] + name: str | None = None + probability: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ChoiceProbability(LiteLLMPydanticObjectBase): + value: ChoiceValue + probability: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ChoiceAnswer(LiteLLMPydanticObjectBase): + type: Literal["choice"] + name: str | None = None + choice: ChoiceValue + probabilities: Sequence[ChoiceProbability] + confidence: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ScoreProbability(LiteLLMPydanticObjectBase): + value: int + label: str + probability: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class ScoreAnswer(LiteLLMPydanticObjectBase): + type: Literal["score"] + name: str | None = None + score: float + probabilities: Sequence[ScoreProbability] + confidence: float + + model_config = ConfigDict(extra="allow", frozen=True) + + +class RefusalAnswer(LiteLLMPydanticObjectBase): + type: Literal["refusal"] + name: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +DecisionAnswer: TypeAlias = Annotated[ + PredicateAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer, + Field(discriminator="type"), +] + + +class DecisionsInputTokensDetails(LiteLLMPydanticObjectBase): + cached_tokens: int = 0 + cache_write_tokens: int = 0 + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsOutputTokensDetails(LiteLLMPydanticObjectBase): + reasoning_tokens: int = 0 + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsUsage(LiteLLMPydanticObjectBase): + input_tokens: int = 0 + input_tokens_details: DecisionsInputTokensDetails = Field(default_factory=DecisionsInputTokensDetails) + output_tokens: int = 0 + output_tokens_details: DecisionsOutputTokensDetails = Field(default_factory=DecisionsOutputTokensDetails) + total_tokens: int = 0 + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsResponse(LiteLLMPydanticObjectBase): + model: str | None = None + answers: Sequence[DecisionAnswer] + usage: DecisionsUsage | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + @property + def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + return self._hidden_params + + def set_hidden_params(self, params: Mapping[str, object]) -> None: + self._hidden_params.update(params) diff --git a/tests/unit/llms/base_llm/decisions/__init__.py b/tests/unit/llms/base_llm/decisions/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/unit/llms/base_llm/decisions/test_systemone.py b/tests/unit/llms/base_llm/decisions/test_systemone.py new file mode 100644 index 000000000000..0c3e02c2ab5e --- /dev/null +++ b/tests/unit/llms/base_llm/decisions/test_systemone.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.systemone import ( + SYSTEM_ONE_RESPONSE_ADAPTER, + decisions_response, + question_keys, + system_one_request, +) +from litellm.types.openai_decisions import ChoiceAnswer, DecisionsRequest, PredicateAnswer, ScoreAnswer + +_INPUT: Final = "The export job hangs at 99% and never finishes" +_QUESTIONS: Final[Sequence[Mapping[str, object]]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "How does the customer feel?", + "choices": [{"value": "positive"}, {"value": "negative", "description": "unhappy"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is it?", + "levels": [{"label": "none"}, {"label": "low"}, {"label": "high", "description": "blocks users"}], + }, +) +_SYSTEM_ONE_QUESTIONS: Final[Mapping[str, object]] = { + "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, + "sentiment": { + "type": "choice", + "instructions": "How does the customer feel?", + "criteria": {"positive": None, "negative": "unhappy"}, + }, + "severity": {"type": "score", "instructions": "How severe is it?", "criteria": ["none", "low", "blocks users"]}, +} +_SYSTEM_ONE_RESPONSE: Final[Mapping[str, object]] = { + "model": "jev-1.13", + "answers": { + "is_defect": {"type": "noul", "noul": 0.9}, + "sentiment": { + "type": "choice", + "choice": "positive", + "confidence": 0.8, + "probabilities": {"positive": 0.8, "negative": 0.2}, + }, + "severity": { + "type": "score", + "score": 1, + "confidence": 0.7, + "legend": {"0": "none", "1": "low", "2": "high"}, + "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, + }, + }, + "usage": {"input_tokens": 367, "output_tokens": 3}, +} +_EXPECTED_ANSWERS: Final[Sequence[Mapping[str, object]]] = ( + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + { + "type": "choice", + "name": "sentiment", + "choice": "positive", + "probabilities": [{"value": "positive", "probability": 0.8}, {"value": "negative", "probability": 0.2}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 1.0, + "probabilities": [ + {"value": 0, "label": "none", "probability": 0.1}, + {"value": 1, "label": "low", "probability": 0.8}, + {"value": 2, "label": "high", "probability": 0.1}, + ], + "confidence": 0.7, + }, +) +_EXPECTED_USAGE: Final[Mapping[str, object]] = { + "input_tokens": 367, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 370, +} +_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest) + + +def _request( + input_value: object = _INPUT, + questions: Sequence[Mapping[str, object]] = _QUESTIONS, +) -> DecisionsRequest: + return _REQUEST_ADAPTER.validate_python({"model": "jev-1.13", "input": input_value, "questions": questions}) + + +def _predicate(name: str | None = "is_defect") -> tuple[Mapping[str, object]]: + return ({"type": "predicate", "name": name, "instructions": "Is this a defect?"},) + + +def test_openai_request_becomes_the_system_one_body() -> None: + assert system_one_request("jev-1.13", _request(), "typesafe") == { + "model": "jev-1.13", + "state": _INPUT, + "questions": _SYSTEM_ONE_QUESTIONS, + } + + +def test_system_one_answers_become_openai_answers_in_question_order() -> None: + response: Final = decisions_response( + SYSTEM_ONE_RESPONSE_ADAPTER.validate_python(_SYSTEM_ONE_RESPONSE), _request(), "typesafe" + ) + + assert response.model_dump(mode="json") == { + "model": "jev-1.13", + "answers": list(_EXPECTED_ANSWERS), + "usage": _EXPECTED_USAGE, + } + assert isinstance(response.answers[0], PredicateAnswer) + assert isinstance(response.answers[1], ChoiceAnswer) + assert isinstance(response.answers[2], ScoreAnswer) + + +def test_user_messages_are_joined_into_one_system_one_state() -> None: + messages: Final = [ + {"role": "user", "content": "first"}, + { + "role": "user", + "content": [{"type": "input_text", "text": "second"}, {"type": "input_text", "text": "third"}], + }, + ] + + body: Final = system_one_request("jev-1.13", _request(messages, _predicate()), "typesafe") + + assert body["state"] == "first\nsecond\nthird" + + +@pytest.mark.parametrize( + ("label", "input_value", "questions"), + ( + ( + "input_image", + [{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]}], + _predicate(), + ), + ("boolean choice", _INPUT, [{"type": "choice", "instructions": "Refund?", "choices": [{"value": True}]}]), + ("unique name", _INPUT, [*_predicate(), *_predicate()]), + ( + "repeated choice", + _INPUT, + [{"type": "choice", "instructions": "Refund?", "choices": [{"value": "yes"}, {"value": "yes"}]}], + ), + ), +) +def test_what_system_one_cannot_express_is_a_400( + label: str, + input_value: object, + questions: Sequence[Mapping[str, object]], +) -> None: + with pytest.raises(BaseLLMException, match=label) as error: + system_one_request("jev-1.13", _request(input_value, questions), "perplexity") + + assert error.value.status_code == 400 + assert "perplexity" in error.value.message + + +def test_unnamed_questions_get_positional_keys_that_never_shadow_a_supplied_name() -> None: + request: Final = _request(questions=[*_predicate(None), *_predicate("q0"), *_predicate(None)]) + + assert question_keys(request.questions, "typesafe") == ("_q0", "q0", "q2") + noul: Final = {"type": "noul", "instructions": "Is this a defect?"} + assert system_one_request("jev-1.13", request, "typesafe")["questions"] == {"_q0": noul, "q0": noul, "q2": noul} + + +def test_positional_answers_come_back_in_question_order_without_a_name() -> None: + request: Final = _request(questions=[*_predicate(None), *_predicate("q0")]) + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( + {"answers": {"_q0": {"type": "noul", "noul": 0.25}, "q0": {"type": "noul", "noul": 0.75}}} + ) + + response: Final = decisions_response(system_one, request, "typesafe") + + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": None, "probability": 0.25}, + {"type": "predicate", "name": "q0", "probability": 0.75}, + ] + assert response.usage is not None + assert response.usage.model_dump(mode="json") == { + **_EXPECTED_USAGE, + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + } + + +def test_a_choice_the_provider_left_out_of_probabilities_is_reported_at_zero() -> None: + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( + { + "answers": { + "sentiment": { + "type": "choice", + "choice": "positive", + "confidence": 1.0, + "probabilities": {"positive": 1.0}, + } + } + } + ) + request: Final = _request(questions=_QUESTIONS[1:2]) + + response: Final = decisions_response(system_one, request, "typesafe") + + assert response.answers[0].model_dump(mode="json") == { + "type": "choice", + "name": "sentiment", + "choice": "positive", + "probabilities": [{"value": "positive", "probability": 1.0}, {"value": "negative", "probability": 0.0}], + "confidence": 1.0, + } + + +@pytest.mark.parametrize( + "answers", + ( + {}, + {"is_defect": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0}}}, + ), +) +def test_a_reply_without_a_matching_answer_is_a_server_error(answers: Mapping[str, object]) -> None: + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python({"answers": answers}) + + with pytest.raises(BaseLLMException, match="no predicate answer for question 'is_defect'") as error: + decisions_response(system_one, _request(questions=_predicate()), "typesafe") + + assert error.value.status_code == 500 diff --git a/tests/unit/types/test_openai_decisions.py b/tests/unit/types/test_openai_decisions.py new file mode 100644 index 000000000000..6b5139e4cdec --- /dev/null +++ b/tests/unit/types/test_openai_decisions.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +import pytest +from pydantic import TypeAdapter, ValidationError + +from litellm.types.openai_decisions import ( + ChoiceAnswer, + DecisionsRequestBody, + DecisionsResponse, + RefusalAnswer, + ScoreQuestion, +) + +_REQUEST: Final[Mapping[str, object]] = { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Is this receipt a valid business expense?"}, + {"type": "input_image", "image_url": "https://example.com/receipt.png", "detail": "high"}, + ], + } + ], + "questions": [ + {"type": "predicate", "name": "is_expense", "instructions": "Is this a business expense?"}, + { + "type": "choice", + "name": "approve", + "instructions": "Should this be approved?", + "choices": [{"value": True, "description": "approve"}, {"value": False, "description": "reject"}], + }, + { + "type": "score", + "name": "risk", + "instructions": "How risky is this expense?", + "levels": [{"label": "low"}, {"label": "high", "description": "needs a manager"}], + }, + ], + "safety_identifier": "user-123", +} +_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "is_expense", "probability": 0.92}, + { + "type": "choice", + "name": "approve", + "choice": True, + "probabilities": [{"value": True, "probability": 0.7}, {"value": False, "probability": 0.3}], + "confidence": 0.7, + }, + {"type": "refusal", "name": "risk"}, + ], + "usage": { + "input_tokens": 120, + "input_tokens_details": {"cached_tokens": 100, "cache_write_tokens": 0}, + "output_tokens": 12, + "output_tokens_details": {"reasoning_tokens": 4}, + "total_tokens": 132, + }, +} +_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) + + +def test_the_documented_request_round_trips_with_its_boolean_choices_and_image_part() -> None: + request: Final = _REQUEST_ADAPTER.validate_python(_REQUEST) + + assert request.model_dump(mode="json", exclude_none=True) == _REQUEST + assert isinstance(request.questions[2], ScoreQuestion) + + +def test_the_documented_response_keeps_answer_order_refusals_and_token_details() -> None: + response: Final = _RESPONSE_ADAPTER.validate_python(_RESPONSE) + + assert response.model_dump(mode="json") == _RESPONSE + assert isinstance(response.answers[1], ChoiceAnswer) + assert isinstance(response.answers[2], RefusalAnswer) + + +_OFF_SPEC: Final[tuple[tuple[str, object], ...]] = ( + ("questions", ()), + ("questions", [{"type": "noul", "name": "q", "instructions": "x"}]), + ("questions", [{"type": "choice", "name": "q", "instructions": "x", "choices": ()}]), + ("questions", [{"type": "score", "name": "q", "levels": [{"label": "low"}]}]), + ("input", {"state": "not an OpenAI input"}), +) + + +@pytest.mark.parametrize(("field", "value"), _OFF_SPEC) +def test_requests_off_the_spec_are_rejected(field: str, value: object) -> None: + with pytest.raises(ValidationError): + _REQUEST_ADAPTER.validate_python({**_REQUEST, field: value}) + + +def test_hidden_params_live_outside_the_wire_body() -> None: + response: Final = _RESPONSE_ADAPTER.validate_python(_RESPONSE) + + response.set_hidden_params({"custom_llm_provider": "openai"}) + + assert response.hidden_params == {"custom_llm_provider": "openai"} + assert "_hidden_params" not in response.model_dump(mode="json") From 09be31552a98a1b1457abd4335840dc208ba0a59 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 20:56:18 +0000 Subject: [PATCH 02/15] refactor(decisions): share one DecisionsModel config and require model and usage on responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/base_llm/decisions/systemone.py | 6 +- litellm/types/openai_decisions.py | 100 ++++++------------ .../llms/base_llm/decisions/test_systemone.py | 15 ++- tests/unit/types/test_openai_decisions.py | 16 +++ 4 files changed, 68 insertions(+), 69 deletions(-) diff --git a/litellm/llms/base_llm/decisions/systemone.py b/litellm/llms/base_llm/decisions/systemone.py index 699927924005..34b73d728554 100644 --- a/litellm/llms/base_llm/decisions/systemone.py +++ b/litellm/llms/base_llm/decisions/systemone.py @@ -22,6 +22,8 @@ DecisionInputMessage, DecisionInputPart, DecisionQuestion, + DecisionsInputTokensDetails, + DecisionsOutputTokensDetails, DecisionsRequest, DecisionsResponse, DecisionsUsage, @@ -198,14 +200,16 @@ def decisions_response( keys: Final = question_keys(request.questions, custom_llm_provider) usage: Final = system_one.usage if system_one.usage is not None else SystemOneUsage() return DecisionsResponse( - model=system_one.model, + model=system_one.model if system_one.model is not None else request.model, answers=[ _answer_for(key, question, system_one.answers, custom_llm_provider) for key, question in zip(keys, request.questions, strict=True) ], usage=DecisionsUsage( input_tokens=usage.input_tokens, + input_tokens_details=DecisionsInputTokensDetails(cached_tokens=0, cache_write_tokens=0), output_tokens=usage.output_tokens, + output_tokens_details=DecisionsOutputTokensDetails(reasoning_tokens=0), total_tokens=usage.input_tokens + usage.output_tokens, ), ) diff --git a/litellm/types/openai_decisions.py b/litellm/types/openai_decisions.py index 6fcbe9ff61e8..d38838898f3b 100644 --- a/litellm/types/openai_decisions.py +++ b/litellm/types/openai_decisions.py @@ -9,74 +9,62 @@ ChoiceValue: TypeAlias = str | bool -class DecisionInputText(LiteLLMPydanticObjectBase): +class DecisionsModel(LiteLLMPydanticObjectBase): + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionInputText(DecisionsModel): type: Literal["input_text"] text: str - model_config = ConfigDict(extra="allow", frozen=True) - -class DecisionInputImage(LiteLLMPydanticObjectBase): +class DecisionInputImage(DecisionsModel): type: Literal["input_image"] image_url: str detail: Literal["low", "high", "auto", "original"] | None = None - model_config = ConfigDict(extra="allow", frozen=True) - DecisionInputPart: TypeAlias = Annotated[DecisionInputText | DecisionInputImage, Field(discriminator="type")] -class DecisionInputMessage(LiteLLMPydanticObjectBase): +class DecisionInputMessage(DecisionsModel): role: Literal["user"] content: str | Sequence[DecisionInputPart] type: Literal["message"] | None = None - model_config = ConfigDict(extra="allow", frozen=True) - DecisionsInput: TypeAlias = str | Sequence[DecisionInputMessage] -class DecisionChoice(LiteLLMPydanticObjectBase): +class DecisionChoice(DecisionsModel): value: ChoiceValue description: str | None = None - model_config = ConfigDict(extra="allow", frozen=True) - -class DecisionLevel(LiteLLMPydanticObjectBase): +class DecisionLevel(DecisionsModel): label: str description: str | None = None - model_config = ConfigDict(extra="allow", frozen=True) - -class PredicateQuestion(LiteLLMPydanticObjectBase): +class PredicateQuestion(DecisionsModel): type: Literal["predicate"] instructions: str name: str | None = None - model_config = ConfigDict(extra="allow", frozen=True) - -class ChoiceQuestion(LiteLLMPydanticObjectBase): +class ChoiceQuestion(DecisionsModel): type: Literal["choice"] instructions: str choices: Annotated[Sequence[DecisionChoice], Field(min_length=1)] name: str | None = None - model_config = ConfigDict(extra="allow", frozen=True) - -class ScoreQuestion(LiteLLMPydanticObjectBase): +class ScoreQuestion(DecisionsModel): type: Literal["score"] instructions: str levels: Annotated[Sequence[DecisionLevel], Field(min_length=1)] name: str | None = None - model_config = ConfigDict(extra="allow", frozen=True) - DecisionQuestion: TypeAlias = Annotated[ PredicateQuestion | ChoiceQuestion | ScoreQuestion, @@ -86,13 +74,11 @@ class ScoreQuestion(LiteLLMPydanticObjectBase): DecisionQuestions: TypeAlias = Annotated[Sequence[DecisionQuestion], Field(min_length=1)] -class DecisionsRequestBody(LiteLLMPydanticObjectBase): +class DecisionsRequestBody(DecisionsModel): input: DecisionsInput questions: DecisionQuestions safety_identifier: str | None = None - model_config = ConfigDict(extra="allow", frozen=True) - class DecisionsRequest(DecisionsRequestBody): model: str @@ -111,55 +97,43 @@ class DecisionsCallParams(TypedDict, total=False): extra_headers: ReadOnly[Mapping[str, str] | None] -class PredicateAnswer(LiteLLMPydanticObjectBase): +class PredicateAnswer(DecisionsModel): type: Literal["predicate"] name: str | None = None probability: float - model_config = ConfigDict(extra="allow", frozen=True) - -class ChoiceProbability(LiteLLMPydanticObjectBase): +class ChoiceProbability(DecisionsModel): value: ChoiceValue probability: float - model_config = ConfigDict(extra="allow", frozen=True) - -class ChoiceAnswer(LiteLLMPydanticObjectBase): +class ChoiceAnswer(DecisionsModel): type: Literal["choice"] name: str | None = None choice: ChoiceValue probabilities: Sequence[ChoiceProbability] confidence: float - model_config = ConfigDict(extra="allow", frozen=True) - -class ScoreProbability(LiteLLMPydanticObjectBase): +class ScoreProbability(DecisionsModel): value: int label: str probability: float - model_config = ConfigDict(extra="allow", frozen=True) - -class ScoreAnswer(LiteLLMPydanticObjectBase): +class ScoreAnswer(DecisionsModel): type: Literal["score"] name: str | None = None score: float probabilities: Sequence[ScoreProbability] confidence: float - model_config = ConfigDict(extra="allow", frozen=True) - -class RefusalAnswer(LiteLLMPydanticObjectBase): +class RefusalAnswer(DecisionsModel): type: Literal["refusal"] name: str | None = None - model_config = ConfigDict(extra="allow", frozen=True) - DecisionAnswer: TypeAlias = Annotated[ PredicateAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer, @@ -167,35 +141,27 @@ class RefusalAnswer(LiteLLMPydanticObjectBase): ] -class DecisionsInputTokensDetails(LiteLLMPydanticObjectBase): - cached_tokens: int = 0 - cache_write_tokens: int = 0 - - model_config = ConfigDict(extra="allow", frozen=True) - - -class DecisionsOutputTokensDetails(LiteLLMPydanticObjectBase): - reasoning_tokens: int = 0 +class DecisionsInputTokensDetails(DecisionsModel): + cached_tokens: int + cache_write_tokens: int - model_config = ConfigDict(extra="allow", frozen=True) +class DecisionsOutputTokensDetails(DecisionsModel): + reasoning_tokens: int -class DecisionsUsage(LiteLLMPydanticObjectBase): - input_tokens: int = 0 - input_tokens_details: DecisionsInputTokensDetails = Field(default_factory=DecisionsInputTokensDetails) - output_tokens: int = 0 - output_tokens_details: DecisionsOutputTokensDetails = Field(default_factory=DecisionsOutputTokensDetails) - total_tokens: int = 0 - model_config = ConfigDict(extra="allow", frozen=True) +class DecisionsUsage(DecisionsModel): + input_tokens: int + input_tokens_details: DecisionsInputTokensDetails + output_tokens: int + output_tokens_details: DecisionsOutputTokensDetails + total_tokens: int -class DecisionsResponse(LiteLLMPydanticObjectBase): - model: str | None = None +class DecisionsResponse(DecisionsModel): + model: str answers: Sequence[DecisionAnswer] - usage: DecisionsUsage | None = None - - model_config = ConfigDict(extra="allow", frozen=True) + usage: DecisionsUsage _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) diff --git a/tests/unit/llms/base_llm/decisions/test_systemone.py b/tests/unit/llms/base_llm/decisions/test_systemone.py index 0c3e02c2ab5e..f1c7177e83c7 100644 --- a/tests/unit/llms/base_llm/decisions/test_systemone.py +++ b/tests/unit/llms/base_llm/decisions/test_systemone.py @@ -188,7 +188,6 @@ def test_positional_answers_come_back_in_question_order_without_a_name() -> None {"type": "predicate", "name": None, "probability": 0.25}, {"type": "predicate", "name": "q0", "probability": 0.75}, ] - assert response.usage is not None assert response.usage.model_dump(mode="json") == { **_EXPECTED_USAGE, "input_tokens": 0, @@ -197,6 +196,20 @@ def test_positional_answers_come_back_in_question_order_without_a_name() -> None } +def test_a_reply_without_a_model_reports_the_requested_model() -> None: + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( + {k: v for k, v in _SYSTEM_ONE_RESPONSE.items() if k != "model"} + ) + + request: Final = _REQUEST_ADAPTER.validate_python( + {"model": "typesafe/jev-1.13.0", "input": _INPUT, "questions": _QUESTIONS} + ) + + response: Final = decisions_response(system_one, request, "typesafe") + + assert response.model == "typesafe/jev-1.13.0" + + def test_a_choice_the_provider_left_out_of_probabilities_is_reported_at_zero() -> None: system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python( { diff --git a/tests/unit/types/test_openai_decisions.py b/tests/unit/types/test_openai_decisions.py index 6b5139e4cdec..4cbd4212dcb7 100644 --- a/tests/unit/types/test_openai_decisions.py +++ b/tests/unit/types/test_openai_decisions.py @@ -96,6 +96,22 @@ def test_requests_off_the_spec_are_rejected(field: str, value: object) -> None: _REQUEST_ADAPTER.validate_python({**_REQUEST, field: value}) +_OFF_SPEC_RESPONSE: Final[tuple[str, ...]] = ("model", "usage") + + +@pytest.mark.parametrize("field", _OFF_SPEC_RESPONSE) +def test_responses_missing_a_required_field_are_rejected(field: str) -> None: + with pytest.raises(ValidationError): + _RESPONSE_ADAPTER.validate_python({k: v for k, v in _RESPONSE.items() if k != field}) + + +def test_usage_without_token_details_is_rejected() -> None: + usage: Final = {"input_tokens": 120, "output_tokens": 12, "total_tokens": 132} + + with pytest.raises(ValidationError): + _RESPONSE_ADAPTER.validate_python({**_RESPONSE, "usage": usage}) + + def test_hidden_params_live_outside_the_wire_body() -> None: response: Final = _RESPONSE_ADAPTER.validate_python(_RESPONSE) From 7e20e922795a868bdfe49981527f31af82e23867 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 21:10:05 +0000 Subject: [PATCH 03/15] refactor(decisions): rename the shared pydantic parent to DecisionsObjectBase Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/openai_decisions.py | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/litellm/types/openai_decisions.py b/litellm/types/openai_decisions.py index d38838898f3b..39e470d77fea 100644 --- a/litellm/types/openai_decisions.py +++ b/litellm/types/openai_decisions.py @@ -9,16 +9,16 @@ ChoiceValue: TypeAlias = str | bool -class DecisionsModel(LiteLLMPydanticObjectBase): +class DecisionsObjectBase(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="allow", frozen=True) -class DecisionInputText(DecisionsModel): +class DecisionInputText(DecisionsObjectBase): type: Literal["input_text"] text: str -class DecisionInputImage(DecisionsModel): +class DecisionInputImage(DecisionsObjectBase): type: Literal["input_image"] image_url: str detail: Literal["low", "high", "auto", "original"] | None = None @@ -27,7 +27,7 @@ class DecisionInputImage(DecisionsModel): DecisionInputPart: TypeAlias = Annotated[DecisionInputText | DecisionInputImage, Field(discriminator="type")] -class DecisionInputMessage(DecisionsModel): +class DecisionInputMessage(DecisionsObjectBase): role: Literal["user"] content: str | Sequence[DecisionInputPart] type: Literal["message"] | None = None @@ -36,30 +36,30 @@ class DecisionInputMessage(DecisionsModel): DecisionsInput: TypeAlias = str | Sequence[DecisionInputMessage] -class DecisionChoice(DecisionsModel): +class DecisionChoice(DecisionsObjectBase): value: ChoiceValue description: str | None = None -class DecisionLevel(DecisionsModel): +class DecisionLevel(DecisionsObjectBase): label: str description: str | None = None -class PredicateQuestion(DecisionsModel): +class PredicateQuestion(DecisionsObjectBase): type: Literal["predicate"] instructions: str name: str | None = None -class ChoiceQuestion(DecisionsModel): +class ChoiceQuestion(DecisionsObjectBase): type: Literal["choice"] instructions: str choices: Annotated[Sequence[DecisionChoice], Field(min_length=1)] name: str | None = None -class ScoreQuestion(DecisionsModel): +class ScoreQuestion(DecisionsObjectBase): type: Literal["score"] instructions: str levels: Annotated[Sequence[DecisionLevel], Field(min_length=1)] @@ -74,7 +74,7 @@ class ScoreQuestion(DecisionsModel): DecisionQuestions: TypeAlias = Annotated[Sequence[DecisionQuestion], Field(min_length=1)] -class DecisionsRequestBody(DecisionsModel): +class DecisionsRequestBody(DecisionsObjectBase): input: DecisionsInput questions: DecisionQuestions safety_identifier: str | None = None @@ -97,18 +97,18 @@ class DecisionsCallParams(TypedDict, total=False): extra_headers: ReadOnly[Mapping[str, str] | None] -class PredicateAnswer(DecisionsModel): +class PredicateAnswer(DecisionsObjectBase): type: Literal["predicate"] name: str | None = None probability: float -class ChoiceProbability(DecisionsModel): +class ChoiceProbability(DecisionsObjectBase): value: ChoiceValue probability: float -class ChoiceAnswer(DecisionsModel): +class ChoiceAnswer(DecisionsObjectBase): type: Literal["choice"] name: str | None = None choice: ChoiceValue @@ -116,13 +116,13 @@ class ChoiceAnswer(DecisionsModel): confidence: float -class ScoreProbability(DecisionsModel): +class ScoreProbability(DecisionsObjectBase): value: int label: str probability: float -class ScoreAnswer(DecisionsModel): +class ScoreAnswer(DecisionsObjectBase): type: Literal["score"] name: str | None = None score: float @@ -130,7 +130,7 @@ class ScoreAnswer(DecisionsModel): confidence: float -class RefusalAnswer(DecisionsModel): +class RefusalAnswer(DecisionsObjectBase): type: Literal["refusal"] name: str | None = None @@ -141,16 +141,16 @@ class RefusalAnswer(DecisionsModel): ] -class DecisionsInputTokensDetails(DecisionsModel): +class DecisionsInputTokensDetails(DecisionsObjectBase): cached_tokens: int cache_write_tokens: int -class DecisionsOutputTokensDetails(DecisionsModel): +class DecisionsOutputTokensDetails(DecisionsObjectBase): reasoning_tokens: int -class DecisionsUsage(DecisionsModel): +class DecisionsUsage(DecisionsObjectBase): input_tokens: int input_tokens_details: DecisionsInputTokensDetails output_tokens: int @@ -158,7 +158,7 @@ class DecisionsUsage(DecisionsModel): total_tokens: int -class DecisionsResponse(DecisionsModel): +class DecisionsResponse(DecisionsObjectBase): model: str answers: Sequence[DecisionAnswer] usage: DecisionsUsage From edbd1d8ac75cd7c71987039e1b671985284f7d9e Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 17:44:52 +0000 Subject: [PATCH 04/15] refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 15 + litellm/_lazy_imports_registry.py | 13 + litellm/decisions/main.py | 296 +++++++----------- litellm/llms/base_llm/decisions/__init__.py | 4 +- .../llms/base_llm/decisions/transformation.py | 132 ++++++-- .../cloudflare/decisions/transformation.py | 38 ++- litellm/llms/custom_httpx/llm_http_handler.py | 100 ++++++ .../openrouter/decisions/transformation.py | 16 +- .../perplexity/decisions/transformation.py | 16 +- .../decisions/transformation.py | 14 +- .../llms/typesafe/decisions/transformation.py | 15 +- litellm/types/decisions.py | 3 + litellm/utils.py | 15 + tests/integration/proxy_config.yaml | 24 ++ .../translation/decisions/__init__.py | 0 .../translation/decisions/bases/__init__.py | 0 .../translation/decisions/bases/cloudflare.py | 86 +++++ .../translation/decisions/bases/openrouter.py | 85 +++++ .../translation/decisions/bases/perplexity.py | 81 +++++ .../decisions/bases/strands_decider.py | 37 +++ .../translation/decisions/bases/typesafe.py | 81 +++++ .../translation/decisions/basic/__init__.py | 0 .../basic/test_decisions_basic_cloudflare.py | 11 + .../basic/test_decisions_basic_openrouter.py | 11 + .../basic/test_decisions_basic_perplexity.py | 11 + .../test_decisions_basic_strands_decider.py | 11 + .../basic/test_decisions_basic_typesafe.py | 11 + tests/unit/decisions/test_main.py | 77 ++++- 28 files changed, 919 insertions(+), 284 deletions(-) create mode 100644 tests/integration/translation/decisions/__init__.py create mode 100644 tests/integration/translation/decisions/bases/__init__.py create mode 100644 tests/integration/translation/decisions/bases/cloudflare.py create mode 100644 tests/integration/translation/decisions/bases/openrouter.py create mode 100644 tests/integration/translation/decisions/bases/perplexity.py create mode 100644 tests/integration/translation/decisions/bases/strands_decider.py create mode 100644 tests/integration/translation/decisions/bases/typesafe.py create mode 100644 tests/integration/translation/decisions/basic/__init__.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py diff --git a/litellm/__init__.py b/litellm/__init__.py index c0dc61e2911d..4cc1399ee9e1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1667,6 +1667,21 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None: from .llms.hosted_vllm.rerank.transformation import ( HostedVLLMRerankConfig as HostedVLLMRerankConfig, ) + from .llms.perplexity.decisions.transformation import ( + PerplexityDecisionsConfig as PerplexityDecisionsConfig, + ) + from .llms.typesafe.decisions.transformation import ( + TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, + ) + from .llms.openrouter.decisions.transformation import ( + OpenRouterDecisionsConfig as OpenRouterDecisionsConfig, + ) + from .llms.cloudflare.decisions.transformation import ( + CloudflareDecisionsConfig as CloudflareDecisionsConfig, + ) + from .llms.strands_decider.decisions.transformation import ( + StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 78f5e4ef04b3..f95679684b8f 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -156,6 +156,11 @@ "ScalewayRerankConfig", "DeepinfraRerankConfig", "HostedVLLMRerankConfig", + "PerplexityDecisionsConfig", + "TypeSafeDecisionsConfig", + "OpenRouterDecisionsConfig", + "CloudflareDecisionsConfig", + "StrandsDeciderDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", "VertexAIRerankConfig", @@ -707,6 +712,14 @@ ".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig", ), + "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), + "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), + "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), + "StrandsDeciderDecisionsConfig": ( + ".llms.strands_decider.decisions.transformation", + "StrandsDeciderDecisionsConfig", + ), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig", diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 8304c52a886d..88cce1f070c1 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,6 +1,5 @@ from collections.abc import Mapping from dataclasses import dataclass, field -from types import MappingProxyType from typing import Final import httpx @@ -8,107 +7,86 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig -from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, get_httpx_client -from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT -from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT -from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT -from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT -from litellm.secret_managers.main import get_secret_str +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.decisions import ( DecisionQuestion, DecisionsJSON, DecisionsRequest, DecisionsResponse, ) -from litellm.utils import client - -DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType( - { - "perplexity": PERPLEXITY_DECISIONS_ENDPOINT, - "typesafe": TYPESAFE_DECISIONS_ENDPOINT, - "openrouter": OPENROUTER_DECISIONS_ENDPOINT, - "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, - "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, - } -) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, client _DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest) -_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) -_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_HANDLER: Final = BaseLLMHTTPHandler() @dataclass(frozen=True, slots=True, repr=False) -class _PreparedDecisionsRequest: - config: DecisionsProviderConfig - provider: str - upstream_model: str - url: str +class _DecisionsCall: + model: str + custom_llm_provider: str + provider_config: BaseDecisionsConfig + request: DecisionsRequest + api_base: str api_key: str | None = field(repr=False) - headers: Mapping[str, str] = field(repr=False) - body: Mapping[str, object] = field(repr=False) + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None -def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]: - provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider - if provider not in DECISIONS_ENDPOINTS: - supported: Final = ", ".join(DECISIONS_ENDPOINTS) - raise litellm.BadRequestError( - message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}", - model=model, - llm_provider=provider, - ) - upstream_model: Final = model.removeprefix(f"{provider}/") - if not upstream_model: - raise litellm.BadRequestError( - message="A model name is required for the Decisions API", - model=model, - llm_provider=provider, - ) - return provider, upstream_model +def _supported_providers() -> tuple[str, ...]: + return tuple( + provider.value + for provider in LlmProviders + if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None + ) -def _resolve_api_key( - *, - provider: str, - model: str, - endpoint: DecisionsProviderConfig, - api_key: str | None, -) -> str | None: - if api_key is not None: - return api_key - - server_api_key: Final = next( - (key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key), - None, +def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: + provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) + provider_config: Final = ( + None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider) ) - if server_api_key is None: - if not endpoint.api_key_required: - return None - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", + if provider_config is None: + supported: Final = ", ".join(_supported_providers()) + raise litellm.BadRequestError( + message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}", model=model, - llm_provider=provider, + llm_provider=custom_llm_provider, ) - - return server_api_key + return provider_config -def _prepare_request( +def _prepare_call( *, model: str, state: DecisionsJSON, questions: Mapping[str, DecisionQuestion | Mapping[str, object]], api_key: str | None, api_base: str | None, + timeout: float | httpx.Timeout | None, custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, -) -> _PreparedDecisionsRequest: - provider, upstream_model = _resolve_provider_model(model, custom_llm_provider) + kwargs: Mapping[str, object], +) -> _DecisionsCall: + upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + provider_config: Final = _provider_config(upstream_model, provider) + canonical_model: Final = provider_config.canonical_model(upstream_model) + if not upstream_model: + raise litellm.BadRequestError( + message="A model name is required for the Decisions API", + model=model, + llm_provider=provider, + ) try: - validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( - {"model": model, "state": state, "questions": questions} + request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( + {"model": canonical_model, "state": state, "questions": questions} ) except ValidationError as error: raise litellm.BadRequestError( @@ -117,109 +95,49 @@ def _prepare_request( llm_provider=provider, ) from error - endpoint: Final = DECISIONS_ENDPOINTS[provider] - env_api_base: Final = get_secret_str(endpoint.api_base_env) - default_api_base: Final = endpoint.default_api_base() - resolved_api_base: Final = api_base or env_api_base or default_api_base + resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) if resolved_api_base is None: raise litellm.BadRequestError( - message=endpoint.missing_api_base_message(provider), + message=provider_config.missing_api_base_message(provider), + model=model, + llm_provider=provider, + ) + resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key) + if resolved_api_key is None and provider_config.api_key_required: + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", model=model, llm_provider=provider, ) - resolved_api_key: Final = _resolve_api_key( - provider=provider, - model=model, - endpoint=endpoint, - api_key=api_key, - ) - - canonical_model: Final = endpoint.canonical_model(upstream_model) - outbound_headers: Final = MappingProxyType( - { - **{ - name: value - for name, value in (extra_headers or {}).items() - if name.lower() not in {"authorization", "content-type"} - }, - **({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}), - "Content-Type": "application/json", - } - ) - body: Final = MappingProxyType( - { - "model": endpoint.request_model(canonical_model), - "state": validated_request.state, - "questions": { - name: question.model_dump(mode="json", exclude_none=True) - for name, question in validated_request.questions.items() + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=canonical_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), }, - } - ) - return _PreparedDecisionsRequest( - config=endpoint, - provider=provider, - upstream_model=canonical_model, - url=endpoint.endpoint_url(resolved_api_base, canonical_model), + custom_llm_provider=provider, + ) + return _DecisionsCall( + model=canonical_model, + custom_llm_provider=provider, + provider_config=provider_config, + request=request, + api_base=resolved_api_base, api_key=resolved_api_key, - headers=outbound_headers, - body=body, - ) - - -def _log_request( - prepared: _PreparedDecisionsRequest, - kwargs: Mapping[str, object], -) -> LiteLLMLoggingObj | None: - logging_obj: Final = kwargs.get("litellm_logging_obj") - if not isinstance(logging_obj, LiteLLMLoggingObj): - return None - logging_obj.update_from_kwargs( - kwargs=dict(kwargs), - model=prepared.upstream_model, - litellm_params={ - "litellm_call_id": kwargs.get("litellm_call_id"), - "api_base": prepared.url, - }, - custom_llm_provider=prepared.provider, - ) - request_body: Final = dict(prepared.body) - request_headers: Final = dict(prepared.headers) - logging_obj.pre_call( - input=request_body, - api_key=prepared.api_key, - model=prepared.upstream_model, - additional_args={ - "api_base": prepared.url, - "complete_input_dict": request_body, - "headers": request_headers, - }, - ) - return logging_obj - - -def _parse_response( - response: httpx.Response, - prepared: _PreparedDecisionsRequest, -) -> DecisionsResponse: - response.raise_for_status() - payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content) - result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload)) - result.hidden_params.update( - { - "model": f"{prepared.provider}/{prepared.upstream_model}", - "custom_llm_provider": prepared.provider, - "provider_response_model": f"{prepared.provider}/{prepared.upstream_model}", - } + logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + headers=extra_headers or {}, + timeout=timeout, ) - return result -def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception: +def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception: return litellm.exception_type( - model=f"{prepared.provider}/{prepared.upstream_model}", - custom_llm_provider=prepared.provider, + model=f"{call.custom_llm_provider}/{call.model}", + custom_llm_provider=call.custom_llm_provider, original_exception=error, ) @@ -236,28 +154,31 @@ async def adecisions( extra_headers: Mapping[str, str] | None = None, **kwargs: object, ) -> DecisionsResponse: - prepared: Final = _prepare_request( + call: Final = _prepare_call( model=model, state=state, questions=questions, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) try: - handler: Final = get_async_httpx_client(llm_provider=prepared.provider) - response: Final = await handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, + return await _HANDLER.adecisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.request, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, ) - return _parse_response(response=response, prepared=prepared) except Exception as error: - raise _map_upstream_exception(error, prepared) from error + raise _map_upstream_exception(error, call) from error @client @@ -272,28 +193,31 @@ def decisions( extra_headers: Mapping[str, str] | None = None, **kwargs: object, ) -> DecisionsResponse: - prepared: Final = _prepare_request( + call: Final = _prepare_call( model=model, state=state, questions=questions, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) try: - handler: Final = get_httpx_client() - response: Final = handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, + return _HANDLER.decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.request, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, ) - return _parse_response(response=response, prepared=prepared) except Exception as error: - raise _map_upstream_exception(error, prepared) from error + raise _map_upstream_exception(error, call) from error -__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"] +__all__ = ["adecisions", "decisions"] diff --git a/litellm/llms/base_llm/decisions/__init__.py b/litellm/llms/base_llm/decisions/__init__.py index c18ac9b00f20..313e57e97622 100644 --- a/litellm/llms/base_llm/decisions/__init__.py +++ b/litellm/llms/base_llm/decisions/__init__.py @@ -1,3 +1,3 @@ -from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint +from .transformation import BaseDecisionsConfig -__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"] +__all__ = ["BaseDecisionsConfig"] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index d4fcea247938..ec9646f456de 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,52 +1,116 @@ -from dataclasses import dataclass -from typing import Protocol +from abc import ABC +from collections.abc import Mapping +from typing import Final +import httpx +from pydantic import TypeAdapter, ValidationError -@dataclass(frozen=True, slots=True) -class JevCompatibleDecisionsEndpoint: - default_api_base_value: str | None - path: str - api_key_env: tuple[str, ...] - api_base_env: str - api_key_required: bool = True +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.decisions import DecisionsRequest, DecisionsResponse - def default_api_base(self) -> str | None: - return self.default_api_base_value +PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) +_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) - def missing_api_base_message(self, provider: str) -> str: - return f"api_base is required for Decisions provider '{provider}'" - def canonical_model(self, model: str) -> str: - return model +class BaseDecisionsConfig(ABC): + """A Decisions provider: where to send the request and how to translate it. - def request_model(self, model: str) -> str: - return model + LiteLLM's public /v1/decisions shape is the Jev / System One wire contract, which is also what TypeSafe, + Perplexity, OpenRouter, Cloudflare Clef and Strands Decider speak, so the defaults pass the body through: + POST {api_base}{path} with {"model", "state", "questions"} and a Bearer header when a key is set. Providers + override the hooks whose wire shape differs (path, URL, model naming, response envelope, or the whole request + and response translation). + """ - def endpoint_url(self, api_base: str, model: str) -> str: - return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + path: str = "/v1/systemone" + api_key_env: tuple[str, ...] = () + api_base_env: tuple[str, ...] = () + api_key_required: bool = True - def unwrap_response(self, payload: object) -> object: - return payload + def get_default_api_base(self) -> str | None: + return None + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return f"api_base is required for Decisions provider '{custom_llm_provider}'" -class DecisionsProviderConfig(Protocol): - @property - def api_key_env(self) -> tuple[str, ...]: ... + def resolve_api_base(self, api_base: str | None) -> str | None: + return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base() - @property - def api_base_env(self) -> str: ... + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or self._first_secret(self.api_key_env) - @property - def api_key_required(self) -> bool: ... + @staticmethod + def _first_secret(names: tuple[str, ...]) -> str | None: + return next((value for value in (get_secret_str(name) for name in names) if value), None) - def default_api_base(self) -> str | None: ... + def canonical_model(self, model: str) -> str: + return model + + def request_model(self, model: str) -> str: + return model - def missing_api_base_message(self, provider: str) -> str: ... + def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]: + return { + **{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS}, + **({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}), + "Content-Type": "application/json", + } - def canonical_model(self, model: str) -> str: ... + def get_complete_url(self, api_base: str, model: str) -> str: + return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" - def request_model(self, model: str) -> str: ... + def transform_decisions_request( + self, + model: str, + request: DecisionsRequest, + custom_llm_provider: str, + ) -> dict[str, object]: + return { + "model": self.request_model(model), + "state": request.state, + "questions": { + name: question.model_dump(mode="json", exclude_none=True) + for name, question in request.questions.items() + }, + } - def endpoint_url(self, api_base: str, model: str) -> str: ... + def unwrap_response(self, payload: object) -> object: + return payload - def unwrap_response(self, payload: object) -> object: ... + def transform_decisions_response( + self, + model: str, + custom_llm_provider: str, + raw_response: httpx.Response, + request: DecisionsRequest, + ) -> DecisionsResponse: + payload: Final[object] = PAYLOAD_ADAPTER.validate_json(raw_response.content) + try: + response: Final = _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload)) + except ValidationError as error: + raise BaseLLMException( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}", + ) from error + self.set_hidden_params(response, model, custom_llm_provider) + return response + + @staticmethod + def set_hidden_params(response: DecisionsResponse, model: str, custom_llm_provider: str) -> None: + response.set_hidden_params( + { + "model": f"{custom_llm_provider}/{model}", + "custom_llm_provider": custom_llm_provider, + "provider_response_model": f"{custom_llm_provider}/{model}", + } + ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, + ) -> BaseLLMException: + return BaseLLMException(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 6e8b2999778b..e78a2e61bb7a 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -1,9 +1,9 @@ from collections.abc import Mapping -from dataclasses import dataclass from typing import Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, @@ -12,19 +12,19 @@ _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) -@dataclass(frozen=True, slots=True) -class CloudflareDecisionsEndpoint: - api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",) - api_base_env: str = "CLOUDFLARE_API_BASE" - api_key_required: bool = True +class CloudflareDecisionsConfig(BaseDecisionsConfig): + """Workers AI runs Clef at POST {account}/ai/run/@cf/cloudflare/ and wraps the reply in `result`.""" - def default_api_base(self) -> str | None: + api_key_env = ("CLOUDFLARE_API_KEY",) + api_base_env = ("CLOUDFLARE_API_BASE",) + + def get_default_api_base(self) -> str | None: account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) if account_id is None: return None return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" - def missing_api_base_message(self, provider: str) -> str: + def missing_api_base_message(self, custom_llm_provider: str) -> str: return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" def canonical_model(self, model: str) -> str: @@ -35,7 +35,7 @@ def canonical_model(self, model: str) -> str: def request_model(self, model: str) -> str: return model.rsplit("/", maxsplit=1)[-1] - def endpoint_url(self, api_base: str, model: str) -> str: + def get_complete_url(self, api_base: str, model: str) -> str: normalized_api_base: Final = api_base.rstrip("/") if normalized_api_base.endswith("/ai/v1"): return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" @@ -44,15 +44,13 @@ def endpoint_url(self, api_base: str, model: str) -> str: return f"{normalized_api_base}/ai/run/{model}" def unwrap_response(self, payload: object) -> object: - if not isinstance(payload, Mapping): + try: + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + except ValidationError: return payload - response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) if "answers" in response_mapping: - return payload - result: Final = response_mapping.get("result") - if isinstance(result, Mapping): - return result - return payload - - -CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint() + return response_mapping + try: + return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result")) + except ValidationError: + return response_mapping diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 2b86b9bbe5ce..abdac8317654 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -70,6 +70,7 @@ from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.base_llm.containers.transformation import BaseContainerConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.files.transformation import ( @@ -122,6 +123,7 @@ ContainerObject, DeleteContainerResult, ) +from litellm.types.decisions import DecisionsRequest, DecisionsResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, @@ -1481,6 +1483,103 @@ async def arerank( request_data=request_data, ) + def _prepare_decisions_request( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsRequest, + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + ) -> tuple[str, dict[str, str], dict[str, object]]: + outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key) + url: Final = provider_config.get_complete_url(api_base=api_base, model=model) + data: Final = provider_config.transform_decisions_request( + model=model, request=request, custom_llm_provider=custom_llm_provider + ) + if logging_obj is not None: + logging_obj.pre_call( + input=data, + api_key=api_key, + model=model, + additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers}, + ) + return url, outbound_headers, data + + def decisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsRequest, + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + client: HTTPHandler | None = None, + ) -> DecisionsResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + request=request, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + sync_httpx_client: Final = client if client is not None else get_httpx_client() + try: + response: Final = sync_httpx_client.post( + url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + + async def adecisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsRequest, + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + client: AsyncHTTPHandler | None = None, + ) -> DecisionsResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + request=request, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + async_httpx_client: Final = ( + client + if client is not None + else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider)) + ) + try: + response: Final = await async_httpx_client.post( + url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + def _prepare_audio_transcription_request( self, model: str, @@ -6156,6 +6255,7 @@ def _handle_error( e: Exception, provider_config: Union[ BaseConfig, + BaseDecisionsConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig, diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 7a7466b1239c..53ddce8b4029 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,10 +1,10 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://openrouter.ai/api", - path="/alpha/decisions", - api_key_env=("OPENROUTER_API_KEY",), - api_base_env="OPENROUTER_API_BASE", -) +class OpenRouterDecisionsConfig(BaseDecisionsConfig): + path = "/alpha/decisions" + api_key_env = ("OPENROUTER_API_KEY",) + api_base_env = ("OPENROUTER_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://openrouter.ai/api" diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py index 69a4753f4a38..11e2a38f06ed 100644 --- a/litellm/llms/perplexity/decisions/transformation.py +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -1,10 +1,10 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.perplexity.ai", - path="/v1/decisions", - api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"), - api_base_env="PERPLEXITY_API_BASE", -) +class PerplexityDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY") + api_base_env = ("PERPLEXITY_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.perplexity.ai" diff --git a/litellm/llms/strands_decider/decisions/transformation.py b/litellm/llms/strands_decider/decisions/transformation.py index 265afb2b1489..9ef966e18f2e 100644 --- a/litellm/llms/strands_decider/decisions/transformation.py +++ b/litellm/llms/strands_decider/decisions/transformation.py @@ -1,11 +1,7 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value=None, - path="/v1/systemone", - api_key_env=("STRANDS_DECIDER_API_KEY",), - api_base_env="STRANDS_DECIDER_API_BASE", - api_key_required=False, -) +class StrandsDeciderDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("STRANDS_DECIDER_API_KEY",) + api_base_env = ("STRANDS_DECIDER_API_BASE",) + api_key_required = False diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 17fb24ce4433..e71c526a840f 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -1,10 +1,9 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.typesafe.ai", - path="/v1/systemone", - api_key_env=("TYPESAFE_API_KEY",), - api_base_env="TYPESAFE_API_BASE", -) +class TypeSafeDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("TYPESAFE_API_KEY",) + api_base_env = ("TYPESAFE_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.typesafe.ai" diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 2745e00b6c6f..8bc535ffebe0 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -125,3 +125,6 @@ class DecisionsResponse(LiteLLMPydanticObjectBase): @property def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation return self._hidden_params + + def set_hidden_params(self, params: Mapping[str, object]) -> None: + self._hidden_params.update(params) diff --git a/litellm/utils.py b/litellm/utils.py index 72ce14912a97..7207c6bacaa1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -379,6 +379,7 @@ def _get_cached_audio_utils(): # Type stubs for lazy-loaded config classes and types from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.containers.transformation import BaseContainerConfig + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.llms.base_llm.google_genai.transformation import ( @@ -8961,6 +8962,20 @@ def get_provider_rerank_config( return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() + @staticmethod + def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None: + if provider == LlmProviders.PERPLEXITY: + return litellm.PerplexityDecisionsConfig() + if provider == LlmProviders.TYPESAFE: + return litellm.TypeSafeDecisionsConfig() + if provider == LlmProviders.OPENROUTER: + return litellm.OpenRouterDecisionsConfig() + if provider == LlmProviders.CLOUDFLARE: + return litellm.CloudflareDecisionsConfig() + if provider == LlmProviders.STRANDS_DECIDER: + return litellm.StrandsDeciderDecisionsConfig() + return None + @staticmethod def get_provider_anthropic_messages_config( model: str, diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 6924b641f137..4c0272548382 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -322,6 +322,30 @@ model_list: api_base: http://127.0.0.1:8191 api_key: synthetic-bedrock-mantle-key aws_region_name: us-east-1 + - model_name: perplexity/pplx-decider-v1-27b + litellm_params: + model: perplexity/pplx-decider-v1-27b + api_base: http://127.0.0.1:8191 + api_key: synthetic-perplexity-key + - model_name: typesafe/jev-1.13.0 + litellm_params: + model: typesafe/jev-1.13.0 + api_base: http://127.0.0.1:8191 + api_key: synthetic-typesafe-key + - model_name: openrouter/typesafe/jev-1.13 + litellm_params: + model: openrouter/typesafe/jev-1.13 + api_base: http://127.0.0.1:8191 + api_key: synthetic-openrouter-key + - model_name: cloudflare/@cf/cloudflare/clef + litellm_params: + model: cloudflare/@cf/cloudflare/clef + api_base: http://127.0.0.1:8191 + api_key: synthetic-cloudflare-key + - model_name: strands_decider/strands-decider-2B-hobson-v19 + litellm_params: + model: strands_decider/strands-decider-2B-hobson-v19 + api_base: http://127.0.0.1:8191 general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL diff --git a/tests/integration/translation/decisions/__init__.py b/tests/integration/translation/decisions/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/decisions/bases/__init__.py b/tests/integration/translation/decisions/bases/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py new file mode 100644 index 000000000000..b28d49c39b58 --- /dev/null +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -0,0 +1,86 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Cloudflare Clef: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. + +Provider side from https://developers.cloudflare.com/workers-ai (POST /ai/run/@cf/cloudflare/clef, reply wrapped in result). Mock reply captured live on 2026-10-06. +""" +CLEF_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "cloudflare/@cf/cloudflare/clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", + expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "result": { + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + expected_litellm_response={ + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, +) diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py new file mode 100644 index 000000000000..44ef6c8ac896 --- /dev/null +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -0,0 +1,85 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""OpenRouter: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. + +Provider side from https://openrouter.ai/docs (POST /api/alpha/decisions). Mock reply captured live on 2026-10-06. +""" +TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "openrouter/typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/alpha/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, + expected_litellm_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, +) diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py new file mode 100644 index 000000000000..ccb55b71ccce --- /dev/null +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -0,0 +1,81 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Perplexity: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. + +Provider side from https://docs.perplexity.ai (POST /v1/decisions). Mock reply captured live on 2026-10-06. +""" +PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "perplexity/pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, +) diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py new file mode 100644 index 000000000000..97afcdefd86d --- /dev/null +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -0,0 +1,37 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Strands Decider: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. + +Provider side from https://github.com/strands-agents/decider (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-06. +""" +STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "strands_decider/strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + }, + mock_provider_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, + expected_litellm_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, +) diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py new file mode 100644 index 000000000000..acada7702c30 --- /dev/null +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -0,0 +1,81 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""TypeSafe: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. + +Provider side from https://docs.typesafe.ai (POST /v1/systemone). Mock reply captured live on 2026-10-06. +""" +JEV_1_13_0_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "typesafe/jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, + expected_litellm_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, +) diff --git a/tests/integration/translation/decisions/basic/__init__.py b/tests/integration/translation/decisions/basic/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py new file mode 100644 index 000000000000..79fc0574b3bb --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [CLEF_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py new file mode 100644 index 000000000000..a74cb23f6b43 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.openrouter import TYPESAFE_JEV_1_13_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [TYPESAFE_JEV_1_13_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py new file mode 100644 index 000000000000..e341d24ba537 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.perplexity import PPLX_DECIDER_V1_27B_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [PPLX_DECIDER_V1_27B_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py new file mode 100644 index 000000000000..cae7ca70c780 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.strands_decider import STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py new file mode 100644 index 000000000000..302636e6fb41 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 684248b35618..5994bd416625 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -252,10 +252,7 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: answers={}, usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), ) - response._hidden_params = { - "model": "perplexity/pplx-decider-v1-27b", - "custom_llm_provider": "perplexity", - } + response.set_hidden_params({"model": "perplexity/pplx-decider-v1-27b", "custom_llm_provider": "perplexity"}) cost: Final = litellm.completion_cost(completion_response=response) perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] @@ -308,7 +305,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo @pytest.mark.asyncio async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): + with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): await litellm.adecisions( model="unknown/jev-1.13", state="review", @@ -320,19 +317,81 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo @pytest.mark.asyncio -async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): +async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"): await litellm.adecisions( - model="perplexity/pplx-decider-v1-27b", + model="anthropic/claude-sonnet-4-5", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, api_key="caller-key", - custom_llm_provider="", ) assert len(respx_mock.calls) == 0 +@pytest.mark.asyncio +async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + custom_llm_provider="", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content)["model"] == "pplx-decider-v1-27b" + assert response._hidden_params["custom_llm_provider"] == "perplexity" + + +@pytest.mark.asyncio +async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}} + ) + + with pytest.raises(litellm.InternalServerError, match="unexpected response"): + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + +@pytest.mark.asyncio +async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_already_resolved( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "openrouter/typesafe/jev-1.13", + "api_key": "deployment-key", + "api_base": "https://egress.example/openrouter", + }, + } + ] + ) + upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE) + + await router.adecisions( + model="jev", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert upstream.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(respx_mock.calls[0].request.content)["model"] == "typesafe/jev-1.13" + + @pytest.mark.asyncio async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): From e3fda062a74b04bb2b48606238834d2f006ea9d9 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 19:09:56 +0000 Subject: [PATCH 05/15] fix(decisions): map OpenRouter connection failures to APIConnectionError and drop explanatory docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../exception_mapping_utils.py | 5 +++- .../llms/base_llm/decisions/transformation.py | 9 ------ .../cloudflare/decisions/transformation.py | 2 -- .../test_exception_mapping_utils.py | 28 +++++++++++++++++++ 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 7fb28fb418b9..b5afa962cdc7 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2147,7 +2147,10 @@ def _map_openrouter_exception( exception_provider: str, extra_information: str, ) -> None: - if hasattr(original_exception, "status_code"): + received_status: Final = hasattr(original_exception, "status_code") and not getattr( + original_exception, "status_code_is_synthesized", False + ) + if received_status: if original_exception.status_code == 400: raise BadRequestError( message=f"{exception_provider} - {error_str}", diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index ec9646f456de..36739555cea9 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -15,15 +15,6 @@ class BaseDecisionsConfig(ABC): - """A Decisions provider: where to send the request and how to translate it. - - LiteLLM's public /v1/decisions shape is the Jev / System One wire contract, which is also what TypeSafe, - Perplexity, OpenRouter, Cloudflare Clef and Strands Decider speak, so the defaults pass the body through: - POST {api_base}{path} with {"model", "state", "questions"} and a Bearer header when a key is set. Providers - override the hooks whose wire shape differs (path, URL, model naming, response envelope, or the whole request - and response translation). - """ - path: str = "/v1/systemone" api_key_env: tuple[str, ...] = () api_base_env: tuple[str, ...] = () diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index e78a2e61bb7a..fa8c68ad3a4c 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -13,8 +13,6 @@ class CloudflareDecisionsConfig(BaseDecisionsConfig): - """Workers AI runs Clef at POST {account}/ai/run/@cf/cloudflare/ and wraps the reply in `result`.""" - api_key_env = ("CLOUDFLARE_API_KEY",) api_base_env = ("CLOUDFLARE_API_BASE",) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 486070781ac9..7e7f1b536f89 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1231,6 +1231,34 @@ def test_branchless_provider_transport_error_maps_to_api_connection_error(): ) +def test_openrouter_transport_error_maps_to_api_connection_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="[Errno 111] Connection refused") + original_exception.status_code_is_synthesized = True + + with pytest.raises(litellm.APIConnectionError): + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + + +def test_openrouter_upstream_500_still_maps_to_api_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="upstream exploded") + + with pytest.raises(litellm.APIError) as excinfo: + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + assert excinfo.value.status_code == 500 + + def test_branchless_provider_upstream_500_still_maps_to_internal_server_error(): from litellm.llms.base_llm.chat.transformation import BaseLLMException From 84ec670ed13707f3d5d2aecf0ca823eebbee6e38 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 17:49:43 +0000 Subject: [PATCH 06/15] feat(decisions): switch /v1/decisions to the OpenAI Decisions schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/decisions/main.py | 27 +- .../health_check_helpers.py | 6 +- litellm/llms/base_llm/decisions/systemone.py | 4 +- .../llms/base_llm/decisions/transformation.py | 18 +- litellm/types/decisions.py | 157 ++++-- litellm/types/openai_decisions.py | 173 ------ litellm/utils.py | 8 +- .../cost_calculation/cost_tracking_cases.json | 13 +- .../cost_calculation/test_cost_tracking.py | 51 +- .../management/test_model_health_check.py | 27 +- .../providers/test_decisions_chaos.py | 12 +- .../providers/test_decisions_wire.py | 185 ++++-- .../translation/decisions/bases/cloudflare.py | 51 +- .../translation/decisions/bases/openrouter.py | 53 +- .../translation/decisions/bases/perplexity.py | 54 +- .../decisions/bases/strands_decider.py | 17 +- .../translation/decisions/bases/typesafe.py | 51 +- tests/unit/decisions/test_main.py | 533 ++++++++++-------- .../test_health_check_helpers.py | 4 +- .../litellm_core_utils/test_hidden_params.py | 6 +- .../llms/base_llm/decisions/test_systemone.py | 2 +- .../decisions_endpoints/test_endpoints.py | 36 +- .../test_add_retry_fallback_headers.py | 4 +- ..._openai_decisions.py => test_decisions.py} | 2 +- 24 files changed, 810 insertions(+), 684 deletions(-) delete mode 100644 litellm/types/openai_decisions.py rename tests/unit/types/{test_openai_decisions.py => test_decisions.py} (98%) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 88cce1f070c1..3be2a4946a6c 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from typing import Final @@ -11,7 +11,7 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.decisions import ( DecisionQuestion, - DecisionsJSON, + DecisionsInput, DecisionsRequest, DecisionsResponse, ) @@ -61,8 +61,9 @@ def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfi def _prepare_call( *, model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + input: DecisionsInput | Sequence[Mapping[str, object]], + questions: Sequence[DecisionQuestion | Mapping[str, object]], + safety_identifier: str | None = None, api_key: str | None, api_base: str | None, timeout: float | httpx.Timeout | None, @@ -86,7 +87,7 @@ def _prepare_call( ) try: request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( - {"model": canonical_model, "state": state, "questions": questions} + {"model": canonical_model, "input": input, "questions": questions, "safety_identifier": safety_identifier} ) except ValidationError as error: raise litellm.BadRequestError( @@ -145,8 +146,9 @@ def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception @client async def adecisions( model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + input: DecisionsInput | Sequence[Mapping[str, object]], + questions: Sequence[DecisionQuestion | Mapping[str, object]], + safety_identifier: str | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, @@ -156,8 +158,9 @@ async def adecisions( ) -> DecisionsResponse: call: Final = _prepare_call( model=model, - state=state, + input=input, questions=questions, + safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, timeout=timeout, @@ -184,8 +187,9 @@ async def adecisions( @client def decisions( model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + input: DecisionsInput | Sequence[Mapping[str, object]], + questions: Sequence[DecisionQuestion | Mapping[str, object]], + safety_identifier: str | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, @@ -195,8 +199,9 @@ def decisions( ) -> DecisionsResponse: call: Final = _prepare_call( model=model, - state=state, + input=input, questions=questions, + safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, timeout=timeout, diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 60e4b7495ede..131bea6db265 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -299,8 +299,10 @@ def get_mode_handlers( "evaluation": lambda: litellm.adecisions( **DECISIONS_CALL_PARAMS.validate_python( { - "state": prompt or "health check", - "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}}, + "input": prompt or "health check", + "questions": [ + {"type": "predicate", "name": "reachable", "instructions": "Is the service reachable?"} + ], **_filter_model_params(model_params=model_params), } ) diff --git a/litellm/llms/base_llm/decisions/systemone.py b/litellm/llms/base_llm/decisions/systemone.py index 34b73d728554..69dcddd22922 100644 --- a/litellm/llms/base_llm/decisions/systemone.py +++ b/litellm/llms/base_llm/decisions/systemone.py @@ -12,8 +12,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.types.llms.base import LiteLLMPydanticObjectBase -from litellm.types.openai_decisions import ( +from litellm.types.decisions import ( ChoiceAnswer, ChoiceProbability, ChoiceQuestion, @@ -33,6 +32,7 @@ ScoreProbability, ScoreQuestion, ) +from litellm.types.llms.base import LiteLLMPydanticObjectBase SystemOneJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 36739555cea9..c6de243d07d4 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -6,11 +6,15 @@ from pydantic import TypeAdapter, ValidationError from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.systemone import ( + SYSTEM_ONE_RESPONSE_ADAPTER, + decisions_response, + system_one_request, +) from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import DecisionsRequest, DecisionsResponse PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) -_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) @@ -58,14 +62,7 @@ def transform_decisions_request( request: DecisionsRequest, custom_llm_provider: str, ) -> dict[str, object]: - return { - "model": self.request_model(model), - "state": request.state, - "questions": { - name: question.model_dump(mode="json", exclude_none=True) - for name, question in request.questions.items() - }, - } + return system_one_request(self.request_model(model), request, custom_llm_provider) def unwrap_response(self, payload: object) -> object: return payload @@ -79,12 +76,13 @@ def transform_decisions_response( ) -> DecisionsResponse: payload: Final[object] = PAYLOAD_ADAPTER.validate_json(raw_response.content) try: - response: Final = _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload)) + system_one: Final = SYSTEM_ONE_RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload)) except ValidationError as error: raise BaseLLMException( status_code=500, message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}", ) from error + response: Final = decisions_response(system_one, request, custom_llm_provider) self.set_hidden_params(response, model, custom_llm_provider) return response diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 8bc535ffebe0..39e470d77fea 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -1,61 +1,83 @@ from collections.abc import Mapping, Sequence from typing import Annotated, Literal, TypeAlias -from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config +from pydantic import ConfigDict, Field, PrivateAttr, with_config from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase -DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] -NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None] +ChoiceValue: TypeAlias = str | bool -class NoulQuestion(LiteLLMPydanticObjectBase): - type: Literal["noul"] - instructions: DecisionsJSON | None = None - criteria: NoulCriteria | None = None - +class DecisionsObjectBase(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="allow", frozen=True) - @model_validator(mode="after") - def require_instructions_or_criteria(self) -> "NoulQuestion": - if self.instructions is None and self.criteria is None: - raise ValueError("A noul question requires instructions or criteria") - return self +class DecisionInputText(DecisionsObjectBase): + type: Literal["input_text"] + text: str -class ChoiceQuestion(LiteLLMPydanticObjectBase): - type: Literal["choice"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] - model_config = ConfigDict(extra="allow", frozen=True) +class DecisionInputImage(DecisionsObjectBase): + type: Literal["input_image"] + image_url: str + detail: Literal["low", "high", "auto", "original"] | None = None -class ScoreQuestion(LiteLLMPydanticObjectBase): - type: Literal["score"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] +DecisionInputPart: TypeAlias = Annotated[DecisionInputText | DecisionInputImage, Field(discriminator="type")] - model_config = ConfigDict(extra="allow", frozen=True) + +class DecisionInputMessage(DecisionsObjectBase): + role: Literal["user"] + content: str | Sequence[DecisionInputPart] + type: Literal["message"] | None = None + + +DecisionsInput: TypeAlias = str | Sequence[DecisionInputMessage] + + +class DecisionChoice(DecisionsObjectBase): + value: ChoiceValue + description: str | None = None + + +class DecisionLevel(DecisionsObjectBase): + label: str + description: str | None = None + + +class PredicateQuestion(DecisionsObjectBase): + type: Literal["predicate"] + instructions: str + name: str | None = None + + +class ChoiceQuestion(DecisionsObjectBase): + type: Literal["choice"] + instructions: str + choices: Annotated[Sequence[DecisionChoice], Field(min_length=1)] + name: str | None = None + + +class ScoreQuestion(DecisionsObjectBase): + type: Literal["score"] + instructions: str + levels: Annotated[Sequence[DecisionLevel], Field(min_length=1)] + name: str | None = None DecisionQuestion: TypeAlias = Annotated[ - NoulQuestion | ChoiceQuestion | ScoreQuestion, + PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type"), ] -DecisionQuestionMap: TypeAlias = Annotated[ - Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion], - Field(min_length=1, max_length=128), -] +DecisionQuestions: TypeAlias = Annotated[Sequence[DecisionQuestion], Field(min_length=1)] -class DecisionsRequestBody(LiteLLMPydanticObjectBase): - state: DecisionsJSON - questions: DecisionQuestionMap - - model_config = ConfigDict(extra="allow", frozen=True) +class DecisionsRequestBody(DecisionsObjectBase): + input: DecisionsInput + questions: DecisionQuestions + safety_identifier: str | None = None class DecisionsRequest(DecisionsRequestBody): @@ -65,8 +87,9 @@ class DecisionsRequest(DecisionsRequestBody): @with_config(ConfigDict(extra="allow")) class DecisionsCallParams(TypedDict, total=False): model: Required[ReadOnly[str]] - state: Required[ReadOnly[DecisionsJSON]] - questions: Required[ReadOnly[DecisionQuestionMap]] + input: Required[ReadOnly[DecisionsInput]] + questions: Required[ReadOnly[DecisionQuestions]] + safety_identifier: ReadOnly[str | None] api_key: ReadOnly[str | None] api_base: ReadOnly[str | None] timeout: ReadOnly[float | None] @@ -74,51 +97,71 @@ class DecisionsCallParams(TypedDict, total=False): extra_headers: ReadOnly[Mapping[str, str] | None] -class NoulAnswer(LiteLLMPydanticObjectBase): - type: Literal["noul"] - noul: float +class PredicateAnswer(DecisionsObjectBase): + type: Literal["predicate"] + name: str | None = None + probability: float - model_config = ConfigDict(extra="allow", frozen=True) +class ChoiceProbability(DecisionsObjectBase): + value: ChoiceValue + probability: float -class ChoiceAnswer(LiteLLMPydanticObjectBase): + +class ChoiceAnswer(DecisionsObjectBase): type: Literal["choice"] - choice: str + name: str | None = None + choice: ChoiceValue + probabilities: Sequence[ChoiceProbability] confidence: float - probabilities: Mapping[str, float] - model_config = ConfigDict(extra="allow", frozen=True) +class ScoreProbability(DecisionsObjectBase): + value: int + label: str + probability: float -class ScoreAnswer(LiteLLMPydanticObjectBase): + +class ScoreAnswer(DecisionsObjectBase): type: Literal["score"] + name: str | None = None score: float + probabilities: Sequence[ScoreProbability] confidence: float - legend: Mapping[str, DecisionsJSON] - probabilities: Mapping[str, float] - model_config = ConfigDict(extra="allow", frozen=True) + +class RefusalAnswer(DecisionsObjectBase): + type: Literal["refusal"] + name: str | None = None DecisionAnswer: TypeAlias = Annotated[ - NoulAnswer | ChoiceAnswer | ScoreAnswer, + PredicateAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer, Field(discriminator="type"), ] -class DecisionsUsage(LiteLLMPydanticObjectBase): - input_tokens: int = 0 - output_tokens: int = 0 +class DecisionsInputTokensDetails(DecisionsObjectBase): + cached_tokens: int + cache_write_tokens: int - model_config = ConfigDict(extra="allow", frozen=True) +class DecisionsOutputTokensDetails(DecisionsObjectBase): + reasoning_tokens: int -class DecisionsResponse(LiteLLMPydanticObjectBase): - model: str | None = None - answers: Mapping[str, DecisionAnswer] - usage: DecisionsUsage | None = None - model_config = ConfigDict(extra="allow", frozen=True) +class DecisionsUsage(DecisionsObjectBase): + input_tokens: int + input_tokens_details: DecisionsInputTokensDetails + output_tokens: int + output_tokens_details: DecisionsOutputTokensDetails + total_tokens: int + + +class DecisionsResponse(DecisionsObjectBase): + model: str + answers: Sequence[DecisionAnswer] + usage: DecisionsUsage _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) diff --git a/litellm/types/openai_decisions.py b/litellm/types/openai_decisions.py deleted file mode 100644 index 39e470d77fea..000000000000 --- a/litellm/types/openai_decisions.py +++ /dev/null @@ -1,173 +0,0 @@ -from collections.abc import Mapping, Sequence -from typing import Annotated, Literal, TypeAlias - -from pydantic import ConfigDict, Field, PrivateAttr, with_config -from typing_extensions import ReadOnly, Required, TypedDict - -from litellm.types.llms.base import LiteLLMPydanticObjectBase - -ChoiceValue: TypeAlias = str | bool - - -class DecisionsObjectBase(LiteLLMPydanticObjectBase): - model_config = ConfigDict(extra="allow", frozen=True) - - -class DecisionInputText(DecisionsObjectBase): - type: Literal["input_text"] - text: str - - -class DecisionInputImage(DecisionsObjectBase): - type: Literal["input_image"] - image_url: str - detail: Literal["low", "high", "auto", "original"] | None = None - - -DecisionInputPart: TypeAlias = Annotated[DecisionInputText | DecisionInputImage, Field(discriminator="type")] - - -class DecisionInputMessage(DecisionsObjectBase): - role: Literal["user"] - content: str | Sequence[DecisionInputPart] - type: Literal["message"] | None = None - - -DecisionsInput: TypeAlias = str | Sequence[DecisionInputMessage] - - -class DecisionChoice(DecisionsObjectBase): - value: ChoiceValue - description: str | None = None - - -class DecisionLevel(DecisionsObjectBase): - label: str - description: str | None = None - - -class PredicateQuestion(DecisionsObjectBase): - type: Literal["predicate"] - instructions: str - name: str | None = None - - -class ChoiceQuestion(DecisionsObjectBase): - type: Literal["choice"] - instructions: str - choices: Annotated[Sequence[DecisionChoice], Field(min_length=1)] - name: str | None = None - - -class ScoreQuestion(DecisionsObjectBase): - type: Literal["score"] - instructions: str - levels: Annotated[Sequence[DecisionLevel], Field(min_length=1)] - name: str | None = None - - -DecisionQuestion: TypeAlias = Annotated[ - PredicateQuestion | ChoiceQuestion | ScoreQuestion, - Field(discriminator="type"), -] - -DecisionQuestions: TypeAlias = Annotated[Sequence[DecisionQuestion], Field(min_length=1)] - - -class DecisionsRequestBody(DecisionsObjectBase): - input: DecisionsInput - questions: DecisionQuestions - safety_identifier: str | None = None - - -class DecisionsRequest(DecisionsRequestBody): - model: str - - -@with_config(ConfigDict(extra="allow")) -class DecisionsCallParams(TypedDict, total=False): - model: Required[ReadOnly[str]] - input: Required[ReadOnly[DecisionsInput]] - questions: Required[ReadOnly[DecisionQuestions]] - safety_identifier: ReadOnly[str | None] - api_key: ReadOnly[str | None] - api_base: ReadOnly[str | None] - timeout: ReadOnly[float | None] - custom_llm_provider: ReadOnly[str | None] - extra_headers: ReadOnly[Mapping[str, str] | None] - - -class PredicateAnswer(DecisionsObjectBase): - type: Literal["predicate"] - name: str | None = None - probability: float - - -class ChoiceProbability(DecisionsObjectBase): - value: ChoiceValue - probability: float - - -class ChoiceAnswer(DecisionsObjectBase): - type: Literal["choice"] - name: str | None = None - choice: ChoiceValue - probabilities: Sequence[ChoiceProbability] - confidence: float - - -class ScoreProbability(DecisionsObjectBase): - value: int - label: str - probability: float - - -class ScoreAnswer(DecisionsObjectBase): - type: Literal["score"] - name: str | None = None - score: float - probabilities: Sequence[ScoreProbability] - confidence: float - - -class RefusalAnswer(DecisionsObjectBase): - type: Literal["refusal"] - name: str | None = None - - -DecisionAnswer: TypeAlias = Annotated[ - PredicateAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer, - Field(discriminator="type"), -] - - -class DecisionsInputTokensDetails(DecisionsObjectBase): - cached_tokens: int - cache_write_tokens: int - - -class DecisionsOutputTokensDetails(DecisionsObjectBase): - reasoning_tokens: int - - -class DecisionsUsage(DecisionsObjectBase): - input_tokens: int - input_tokens_details: DecisionsInputTokensDetails - output_tokens: int - output_tokens_details: DecisionsOutputTokensDetails - total_tokens: int - - -class DecisionsResponse(DecisionsObjectBase): - model: str - answers: Sequence[DecisionAnswer] - usage: DecisionsUsage - - _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) - - @property - def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation - return self._hidden_params - - def set_hidden_params(self, params: Mapping[str, object]) -> None: - self._hidden_params.update(params) diff --git a/litellm/utils.py b/litellm/utils.py index 7207c6bacaa1..9653d1edbfdd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1219,8 +1219,12 @@ def function_setup( else search_query ) elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value): - decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state", "") - messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state) + decisions_input: Final = args[1] if len(args) > 1 else kwargs.get("input", "") + messages = ( + decisions_input + if isinstance(decisions_input, str) + else json.dumps(decisions_input, default=convert_to_dict) + ) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 94571f2a63cf..b7751d155225 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -31210,15 +31210,14 @@ "endpoint": "/v1/decisions", "request": { "model": "$MODEL", - "state": { - "source": "cost-tracking" - }, - "questions": { - "is_defect": { - "type": "noul", + "input": "cost-tracking", + "questions": [ + { + "type": "predicate", + "name": "is_defect", "instructions": "Is this a defect?" } - } + ] }, "response": { "content_type": "application/json", diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index 379b2c13f5b2..d5d3cc86949c 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -43,10 +43,7 @@ raise ValueError("\n".join(_data_errors)) -_CASES: Final = tuple( - pytest.param(case, marks=pytest.mark.covers(case.covers), id=case.name) - for case in CASES -) +_CASES: Final = tuple(pytest.param(case, marks=pytest.mark.covers(case.covers), id=case.name) for case in CASES) def _wav_bytes(seconds: float) -> bytes: @@ -98,9 +95,9 @@ def _assert_stream_has_no_error(response_text: str) -> None: if payload == "[DONE]": continue parsed = JSON_OBJECT.validate_json(payload) - assert ( - "error" not in parsed and parsed.get("type") not in {"error", "response.failed"} - ), f"stream carried an error event: {parsed}" + assert "error" not in parsed and parsed.get("type") not in {"error", "response.failed"}, ( + f"stream carried an error event: {parsed}" + ) def _replace_model(value: JsonValue, model_name: str) -> JsonValue: @@ -132,11 +129,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) with gateway.scenario() as scenario: expected: Final = case.expected team_id: Final = scenario.team() if isinstance(expected, ExactExpected) and expected.rollups else None - user_id: Final = ( - scenario.user(team_id=team_id) - if team_id is not None - else None - ) + user_id: Final = scenario.user(team_id=team_id) if team_id is not None else None key: Final = ( scenario.key(team_id=team_id, user_id=user_id) if team_id is not None and user_id is not None @@ -145,9 +138,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) passthrough_provider: Final = case.passthrough_provider scenario_id: Final = f"sc-{marker}-{sha256(key.encode()).hexdigest()[:12]}" scenario_handle: Final = ( - register_scenario(scenario_id, case.response) - if passthrough_provider in {"gemini", "anthropic"} - else None + register_scenario(scenario_id, case.response) if passthrough_provider in {"gemini", "anthropic"} else None ) if scenario_handle is not None: scenario.cleanups.callback(delete_scenario, scenario_handle) @@ -171,13 +162,17 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) model_name: Final = ( case.model if passthrough_provider in {"gemini", "anthropic"} - else deployment.model_name if deployment is not None else None + else deployment.model_name + if deployment is not None + else None ) assert model_name is not None request_model: Final = ( case.model.rsplit("/", 1)[-1] if passthrough_provider in {"gemini", "anthropic"} - else fallback_deployment.model_name if fallback_deployment is not None else model_name + else fallback_deployment.model_name + if fallback_deployment is not None + else model_name ) base_request_values: Final = ( _replace_model(case.request, request_model) @@ -185,26 +180,18 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) else {**case.request, "model": model_name} ) end_user_id: Final = ( - f"end-user-{uuid.uuid4()}" - if isinstance(expected, ExactExpected) and expected.rollups - else None + f"end-user-{uuid.uuid4()}" if isinstance(expected, ExactExpected) and expected.rollups else None ) request_headers: Final = ( { "x-pass-x-scripted-scenario": scenario_id, - **( - {"x-goog-api-key": key} - if passthrough_provider == "gemini" - else {} - ), + **({"x-goog-api-key": key} if passthrough_provider == "gemini" else {}), } if passthrough_provider is not None else {} ) request_path: Final = ( - case.endpoint.replace("$MODEL", request_model) - if passthrough_provider is not None - else case.endpoint + case.endpoint.replace("$MODEL", request_model) if passthrough_provider is not None else case.endpoint ) prior_response_id: Final = ( _prime_prior_response(gateway, request_path, base_request_values, key) @@ -219,11 +206,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) if fallback_deployment is not None else {} ), - **( - {"user": end_user_id, "cache": {"no-cache": True}} - if end_user_id is not None - else {} - ), + **({"user": end_user_id, "cache": {"no-cache": True}} if end_user_id is not None else {}), **({"previous_response_id": prior_response_id} if prior_response_id is not None else {}), } ) @@ -285,7 +268,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) "api_key": "", "body": { "model": "pplx-decider-v1-27b", - "state": {"source": "cost-tracking"}, + "state": "cost-tracking", "questions": { "is_defect": { "type": "noul", diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 17bd1c2ebfe3..04d26468e6ae 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -32,9 +32,24 @@ "usage": {"input_tokens": 10, "output_tokens": 1}, }, ) -_CONFIGURED_STATE: Final[dict[str, JsonValue]] = {"ticket": "health probe"} -_CONFIGURED_QUESTIONS: Final[dict[str, JsonValue]] = { - "alive": {"type": "choice", "criteria": {"yes": "the service answers", "no": "the service is down"}} +_CONFIGURED_INPUT: Final = "health probe" +_CONFIGURED_QUESTIONS: Final[list[JsonValue]] = [ + { + "type": "choice", + "name": "alive", + "instructions": "Does the service answer?", + "choices": [ + {"value": "yes", "description": "the service answers"}, + {"value": "no", "description": "the service is down"}, + ], + } +] +_CONFIGURED_UPSTREAM_QUESTIONS: Final[dict[str, JsonValue]] = { + "alive": { + "type": "choice", + "instructions": "Does the service answer?", + "criteria": {"yes": "the service answers", "no": "the service is down"}, + } } @@ -104,7 +119,7 @@ def test_evaluation_mode_health_check_resolves_the_mode_from_the_cost_map_and_se ] -def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway) -> None: +def test_evaluation_mode_health_check_sends_the_configured_input_and_questions(gateway: Gateway) -> None: with gateway.scenario() as scenario: handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _CONFIGURED_PROBE_REPLY) scenario.cleanups.callback(delete_scenario, handle) @@ -113,7 +128,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g api_base=handle.api_base(), model_info={ "mode": "evaluation", - "health_check_params": {"state": _CONFIGURED_STATE, "questions": _CONFIGURED_QUESTIONS}, + "health_check_params": {"input": _CONFIGURED_INPUT, "questions": _CONFIGURED_QUESTIONS}, }, ) report: Final = _health_report(gateway, model) @@ -121,7 +136,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g assert _probes_sent_to(gateway, handle) == [ ( f"/{handle.scenario_id}/v1/systemone", - {"model": "jev-custom", "state": _CONFIGURED_STATE, "questions": _CONFIGURED_QUESTIONS}, + {"model": "jev-custom", "state": _CONFIGURED_INPUT, "questions": _CONFIGURED_UPSTREAM_QUESTIONS}, ) ] diff --git a/tests/integration/providers/test_decisions_chaos.py b/tests/integration/providers/test_decisions_chaos.py index 2e88cdbbc7fa..b7905ae055dd 100644 --- a/tests/integration/providers/test_decisions_chaos.py +++ b/tests/integration/providers/test_decisions_chaos.py @@ -26,7 +26,7 @@ _API_KEY: Final = "synthetic-decisions-key" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") -_QUESTIONS: Final[dict[str, JsonValue]] = {"fine": {"type": "noul", "instructions": "Is the state fine?"}} +_QUESTIONS: Final[list[JsonValue]] = [{"type": "predicate", "name": "fine", "instructions": "Is the state fine?"}] _ROUTES: Final = ("/v1/decisions", "/decisions") @@ -63,13 +63,19 @@ def _marker_of(request: Request) -> str: return state +def _asked_questions(request: Request) -> tuple[str, ...]: + questions: Final = _JSON_OBJECT.validate_json(request.body)["questions"] + assert isinstance(questions, dict), request.body + return tuple(questions) + + def _reply(request: Request) -> Reply: marker: Final = _marker_of(request) if marker.startswith("fail-"): return Reply(status=500, body=json.dumps({"error": {"message": f"scripted outage {marker}"}}).encode()) answer: Final = { "model": f"model-{marker}", - "answers": {"fine": {"type": "noul", "noul": 0.5}}, + "answers": {name: {"type": "noul", "noul": 0.5} for name in _asked_questions(request)}, "usage": {"input_tokens": 12, "output_tokens": 1}, } return Reply(body=json.dumps(answer).encode()) @@ -78,7 +84,7 @@ def _reply(request: Request) -> Reply: async def _send(client: httpx.AsyncClient, key: str, model: str | None, call: _Call) -> _Served: body: Final[dict[str, JsonValue]] = { **({"model": model} if model is not None else {}), - "state": call.marker, + "input": call.marker, "questions": _QUESTIONS, "num_retries": 0, } diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 9878559cb07b..0cf564ff72dd 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -25,14 +25,32 @@ _PASS_THROUGH_AUTHORIZATION: Final = "Bearer customer-held-upstream-key" _PASS_THROUGH_NEIGHBOUR: Final = "decisions-beside-a-pass-through" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) -_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 367, "output_tokens": 3} -_STATE: Final[dict[str, JsonValue]] = {"ticket": "The export job hangs at 99%", "component": "billing"} -_QUESTIONS: Final[dict[str, JsonValue]] = { +_INPUT: Final = "Ticket (billing): The export job hangs at 99%" +_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [{"value": "low", "description": "cosmetic"}, {"value": "high", "description": "blocks users"}], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, +] +_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = { "defect": {"type": "noul", "instructions": "Is this a defect?"}, - "severity": {"type": "choice", "criteria": {"low": "cosmetic", "high": "blocks users"}, "weight": 2}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, } -_ANSWERS: Final[dict[str, JsonValue]] = { +_SYSTEM_ONE_ANSWERS: Final[dict[str, JsonValue]] = { "defect": {"type": "noul", "noul": 0.93}, "severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}}, "confidence": { @@ -43,6 +61,36 @@ "probabilities": {"0": 0.3, "1": 0.7}, }, } +_ANSWERS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "probability": 0.93}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.2}, {"value": "high", "probability": 0.8}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "confidence", + "score": 1.0, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.3}, + {"value": 1, "label": "sure", "probability": 0.7}, + ], + "confidence": 0.7, + }, +] +_INPUT_TOKENS: Final = 367 +_OUTPUT_TOKENS: Final = 3 +_SYSTEM_ONE_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS} +_USAGE: Final[dict[str, JsonValue]] = { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, +} _CHAT_BODY: Final[dict[str, JsonValue]] = {"messages": [{"role": "user", "content": "hi"}]} _CHAT_REPLY: Final[dict[str, JsonValue]] = { "id": "chatcmpl-decisions-parity", @@ -68,6 +116,20 @@ class _Provider: wraps_result: bool cost_map_key: str | None + def upstream_body(self) -> dict[str, JsonValue]: + return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + + def upstream_reply(self) -> dict[str, JsonValue]: + answer: Final[dict[str, JsonValue]] = { + "model": self.body_model, + "answers": _SYSTEM_ONE_ANSWERS, + "usage": _SYSTEM_ONE_USAGE, + } + return {"result": answer, "success": True} if self.wraps_result else answer + + def litellm_response(self) -> dict[str, JsonValue]: + return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + _PROVIDERS: Final = ( _Provider( @@ -103,19 +165,28 @@ class _Provider: ), ) _PERPLEXITY: Final = _PROVIDERS[0] +_PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( - ("missing questions", {"state": _STATE}), - ("missing state", {"questions": _QUESTIONS}), - ("numeric state", {"state": 5, "questions": _QUESTIONS}), - ("empty questions", {"state": _STATE, "questions": {}}), - ("noul without instructions or criteria", {"state": _STATE, "questions": {"q": {"type": "noul"}}}), - ("choice without criteria", {"state": _STATE, "questions": {"q": {"type": "choice", "criteria": {}}}}), - ( - "score with eleven criteria", - {"state": _STATE, "questions": {"q": {"type": "score", "criteria": [f"level-{index}" for index in range(11)]}}}, - ), - ("unknown question type", {"state": _STATE, "questions": {"q": {"type": "ranking", "criteria": ["a"]}}}), + ("missing questions", {"input": _INPUT}), + ("missing input", {"questions": _QUESTIONS}), + ("numeric input", {"input": 5, "questions": _QUESTIONS}), + ("assistant message input", {"input": [{"role": "assistant", "content": "hi"}], "questions": _QUESTIONS}), + ("empty questions", {"input": _INPUT, "questions": []}), + ("questions as a map", {"input": _INPUT, "questions": {"q": _PREDICATE}}), + ("predicate without instructions", {"input": _INPUT, "questions": [{"type": "predicate", "name": "q"}]}), + ("choice without choices", {"input": _INPUT, "questions": [{**_PREDICATE, "type": "choice", "choices": []}]}), + ("score without levels", {"input": _INPUT, "questions": [{**_PREDICATE, "type": "score", "levels": []}]}), + ("unknown question type", {"input": _INPUT, "questions": [{**_PREDICATE, "type": "ranking"}]}), ) +_IMAGE_INPUT: Final[list[JsonValue]] = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": _INPUT}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } +] def _number(value: JsonValue) -> float: @@ -127,14 +198,9 @@ def _expected_spend(cost_map_key: str | None) -> float: if cost_map_key is None: return 0.0 prices: Final = object_value(json.loads(Path("model_prices_and_context_window.json").read_text())[cost_map_key]) - return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number( - _USAGE["output_tokens"] - ) * _number(prices["output_cost_per_token"]) - - -def _answer_body(provider: _Provider) -> dict[str, JsonValue]: - answer: Final[dict[str, JsonValue]] = {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE} - return {"result": answer, "success": True} if provider.wraps_result else answer + return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number( + prices["output_cost_per_token"] + ) def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle: @@ -151,7 +217,7 @@ def _deployment(scenario: Scenario, handle: ScenarioHandle, provider: _Provider) def _decide(gateway: Gateway, model: str, *, key: str | None = None, **extra: JsonValue) -> httpx.Response: return gateway.request( - "POST", "/v1/decisions", {"model": model, "state": _STATE, "questions": _QUESTIONS, **extra}, key=key + "POST", "/v1/decisions", {"model": model, "input": _INPUT, "questions": _QUESTIONS, **extra}, key=key ) @@ -215,17 +281,17 @@ def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cos ) -> None: expected_spend: Final = _expected_spend(provider.cost_map_key) with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(provider)) + handle: Final = _register(scenario, provider.upstream_reply()) model: Final = _deployment(scenario, handle, provider) response: Final = _decide(gateway, model) assert response.status_code == 200, response.text - assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE} + assert response.json() == provider.litellm_response() assert response.headers["x-litellm-model-group"] == model assert math.isclose(float(response.headers.get("x-litellm-response-cost", "0")), expected_spend, rel_tol=1e-9) (call,) = _upstream_calls(gateway, handle) assert call["path"] == f"/{handle.scenario_id}{provider.path}" assert call["authorization"] == (f"Bearer {provider.api_key}" if provider.api_key else "") - assert call["body"] == {"model": provider.body_model, "state": _STATE, "questions": _QUESTIONS} + assert call["body"] == provider.upstream_body() row: Final = _spend_row(response.headers["x-litellm-call-id"]) assert ( row["status"], @@ -235,13 +301,21 @@ def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cos row["api_base"], row["prompt_tokens"], row["completion_tokens"], - ) == ("success", "adecisions", provider.name, model, f"{handle.api_base()}{provider.path}", 367, 3) + ) == ( + "success", + "adecisions", + provider.name, + model, + f"{handle.api_base()}{provider.path}", + _INPUT_TOKENS, + _OUTPUT_TOKENS, + ) assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row def test_repeated_identical_requests_each_reach_the_upstream_and_are_each_billed(gateway: Gateway) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) model: Final = _deployment(scenario, handle, _PERPLEXITY) responses: Final = tuple(_decide(gateway, model) for _ in range(2)) assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses] @@ -255,38 +329,34 @@ def test_repeated_identical_requests_each_reach_the_upstream_and_are_each_billed async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway) -> None: provider: Final = _PROVIDERS[1] with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(provider)) + handle: Final = _register(scenario, provider.upstream_reply()) synchronous: Final = litellm.decisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + model=provider.model, input=_INPUT, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) asynchronous: Final = await litellm.adecisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + model=provider.model, input=_INPUT, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) for response in (synchronous, asynchronous): - assert response.model_dump(mode="json") == { - "model": provider.body_model, - "answers": _ANSWERS, - "usage": _USAGE, - } + assert response.model_dump(mode="json") == provider.litellm_response() calls: Final = _upstream_calls(gateway, handle) assert len(calls) == 2, calls for call in calls: assert call["path"] == f"/{handle.scenario_id}{provider.path}" assert call["authorization"] == f"Bearer {_API_KEY}" - assert call["body"] == {"model": provider.body_model, "state": _STATE, "questions": _QUESTIONS} + assert call["body"] == provider.upstream_body() def test_gateway_only_fields_stay_at_the_gateway_and_tags_reach_the_spend_log(gateway: Gateway) -> None: tag: Final = f"decisions-audit-{uuid.uuid4().hex[:8]}" with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) model: Final = _deployment(scenario, handle, _PERPLEXITY) response: Final = _decide( gateway, model, user="auditor", num_retries=0, temperature=0.2, metadata={"tags": [tag]} ) assert response.status_code == 200, response.text (call,) = _upstream_calls(gateway, handle) - assert call["body"] == {"model": _PERPLEXITY.body_model, "state": _STATE, "questions": _QUESTIONS} + assert call["body"] == _PERPLEXITY.upstream_body() row: Final = _spend_row(response.headers["x-litellm-call-id"]) tags: Final = row["request_tags"] assert isinstance(tags, list) and tag in tags, row @@ -294,7 +364,7 @@ def test_gateway_only_fields_stay_at_the_gateway_and_tags_reach_the_spend_log(ga def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gateway: Gateway) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) model: Final = _deployment(scenario, handle, _PERPLEXITY) for label, body in _INVALID_BODIES: response: Final = gateway.request("POST", "/v1/decisions", {"model": model, **body}) @@ -303,6 +373,19 @@ def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gate assert _upstream_calls(gateway, handle) == [] +@pytest.mark.parametrize("provider", _PROVIDERS[:5], ids=lambda provider: provider.name) +def test_system_one_providers_refuse_images_at_the_gateway_without_an_upstream_call( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + response: Final = _decide(gateway, model, input=_IMAGE_INPUT) + assert response.status_code == 400, response.text + assert "input_image" in response.text + assert _upstream_calls(gateway, handle) == [] + + def test_unknown_model_is_refused_like_chat(gateway: Gateway) -> None: model: Final = f"missing-{uuid.uuid4().hex}" decisions: Final = _decide(gateway, model) @@ -313,10 +396,10 @@ def test_unknown_model_is_refused_like_chat(gateway: Gateway) -> None: def test_key_checks_match_chat(gateway: Gateway) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) model: Final = _deployment(scenario, handle, _PERPLEXITY) anonymous: Final = gateway.client.post( - "/v1/decisions", json={"model": model, "state": _STATE, "questions": _QUESTIONS} + "/v1/decisions", json={"model": model, "input": _INPUT, "questions": _QUESTIONS} ) assert anonymous.status_code == 401, anonymous.text restricted: Final = scenario.key(models=[f"other-{uuid.uuid4().hex}"]) @@ -336,7 +419,7 @@ def test_key_checks_match_chat(gateway: Gateway) -> None: def test_request_body_api_base_is_refused_like_chat_without_an_upstream_call(gateway: Gateway) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) model: Final = _deployment(scenario, handle, _PERPLEXITY) decisions: Final = _decide(gateway, model, api_base=f"http://127.0.0.1:{_free_closed_port()}") chat: Final = _chat(gateway, model, api_base=f"http://127.0.0.1:{_free_closed_port()}") @@ -347,7 +430,7 @@ def test_request_body_api_base_is_refused_like_chat_without_an_upstream_call(gat def test_a_deployment_without_a_key_sends_the_provider_env_key_to_its_configured_api_base(gateway: Gateway) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) model: Final = scenario.model(model=_PERPLEXITY.model, api_base=handle.api_base(), api_key=None) response: Final = _decide(gateway, model) assert response.status_code == 200, response.text @@ -359,8 +442,8 @@ def test_a_deployment_opted_into_client_api_base_sends_decisions_and_chat_to_the gateway: Gateway, ) -> None: with gateway.scenario() as scenario: - configured: Final = _register(scenario, _answer_body(_PERPLEXITY)) - decisions_target: Final = _register(scenario, _answer_body(_PERPLEXITY)) + configured: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + decisions_target: Final = _register(scenario, _PERPLEXITY.upstream_reply()) chat_target: Final = _register(scenario, _CHAT_REPLY) model: Final = scenario.model( model=_PERPLEXITY.model, @@ -389,14 +472,14 @@ def test_a_config_pass_through_at_v1_decisions_keeps_answering_and_the_native_ap pass_through_target: Final = _register( scenario, {"model": _PASS_THROUGH_MODEL, "answers": _ANSWERS, "usage": _USAGE} ) - native_target: Final = _register(scenario, _answer_body(_PERPLEXITY)) + native_target: Final = _register(scenario, _PERPLEXITY.upstream_reply()) config: Final = _pass_through_config( tmp_path, f"{pass_through_target.api_base()}/v1/decisions", native_target.api_base() ) with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: through: Final = _decide(owned.gateway, _PASS_THROUGH_MODEL) native: Final = owned.gateway.request( - "POST", "/decisions", {"model": _PASS_THROUGH_NEIGHBOUR, "state": _STATE, "questions": _QUESTIONS} + "POST", "/decisions", {"model": _PASS_THROUGH_NEIGHBOUR, "input": _INPUT, "questions": _QUESTIONS} ) assert through.status_code == 200, through.text assert through.json() == {"model": _PASS_THROUGH_MODEL, "answers": _ANSWERS, "usage": _USAGE} @@ -443,7 +526,7 @@ def test_upstream_success_without_answers_is_a_gateway_side_server_error(gateway def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) healthy: Final = _deployment(scenario, handle, _PERPLEXITY) dead: Final = scenario.model( model=_PERPLEXITY.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=_API_KEY diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py index b28d49c39b58..84d43a11fc37 100644 --- a/tests/integration/translation/decisions/bases/cloudflare.py +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -2,7 +2,7 @@ from integration.translation.case import TranslationTestCase -"""Cloudflare Clef: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. +"""Cloudflare Workers AI: LiteLLM speaks the OpenAI Decisions shape; the provider speaks Jev / System One. Provider side from https://developers.cloudflare.com/workers-ai (POST /ai/run/@cf/cloudflare/clef, reply wrapped in result). Mock reply captured live on 2026-10-06. """ @@ -11,16 +11,25 @@ litellm_endpoint="/v1/decisions", litellm_request={ "model": "cloudflare/@cf/cloudflare/clef", - "state": "Ticket (billing): The export job hangs at 99% and never finishes", - "questions": { - "defect": {"type": "noul", "instructions": "Is this a defect?"}, - "severity": { + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { "type": "choice", + "name": "severity", "instructions": "How severe is it?", - "criteria": {"low": "cosmetic", "high": "blocks users"}, + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], }, - "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, - }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], "cache": {"no-cache": True}, }, expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", @@ -65,22 +74,32 @@ }, expected_litellm_response={ "model": "clef", - "answers": { - "defect": {"type": "noul", "noul": 0.9345}, - "severity": { + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.9345}, + { "type": "choice", + "name": "severity", "choice": "high", + "probabilities": [{"value": "low", "probability": 0.0509}, {"value": "high", "probability": 0.9491}], "confidence": 0.8067, - "probabilities": {"low": 0.0509, "high": 0.9491}, }, - "confidence": { + { "type": "score", + "name": "confidence", "score": 0.9036, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.0964}, + {"value": 1, "label": "sure", "probability": 0.9036}, + ], "confidence": 0.6515, - "legend": {"0": "unsure", "1": "sure"}, - "probabilities": {"0": 0.0964, "1": 0.9036}, }, + ], + "usage": { + "input_tokens": 290, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 290, }, - "usage": {"input_tokens": 290, "output_tokens": 0}, }, ) diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py index 44ef6c8ac896..f8a06a8efd97 100644 --- a/tests/integration/translation/decisions/bases/openrouter.py +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -2,7 +2,7 @@ from integration.translation.case import TranslationTestCase -"""OpenRouter: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. +"""OpenRouter: LiteLLM speaks the OpenAI Decisions shape; the provider speaks Jev / System One. Provider side from https://openrouter.ai/docs (POST /api/alpha/decisions). Mock reply captured live on 2026-10-06. """ @@ -11,16 +11,25 @@ litellm_endpoint="/v1/decisions", litellm_request={ "model": "openrouter/typesafe/jev-1.13", - "state": "Ticket (billing): The export job hangs at 99% and never finishes", - "questions": { - "defect": {"type": "noul", "instructions": "Is this a defect?"}, - "severity": { + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { "type": "choice", + "name": "severity", "instructions": "How severe is it?", - "criteria": {"low": "cosmetic", "high": "blocks users"}, + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], }, - "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, - }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], "cache": {"no-cache": True}, }, expected_provider_endpoint="/alpha/decisions", @@ -62,24 +71,32 @@ }, expected_litellm_response={ "model": "typesafe/jev-1.13-20260917", - "answers": { - "defect": {"type": "noul", "noul": 0.81}, - "severity": { + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.81}, + { "type": "choice", + "name": "severity", "choice": "high", + "probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}], "confidence": 0.99, - "probabilities": {"low": 0.01, "high": 0.99}, }, - "confidence": { + { "type": "score", + "name": "confidence", "score": 0.5, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.5}, + {"value": 1, "label": "sure", "probability": 0.5}, + ], "confidence": 0, - "legend": {"0": "unsure", "1": "sure"}, - "probabilities": {"0": 0.5, "1": 0.5}, }, + ], + "usage": { + "input_tokens": 377, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 62, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 439, }, - "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, - "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", - "provider": "TypeSafe", }, ) diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py index ccb55b71ccce..9fd227acbea2 100644 --- a/tests/integration/translation/decisions/bases/perplexity.py +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -2,7 +2,7 @@ from integration.translation.case import TranslationTestCase -"""Perplexity: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. +"""Perplexity: LiteLLM speaks the OpenAI Decisions shape; the provider speaks Jev / System One. Provider side from https://docs.perplexity.ai (POST /v1/decisions). Mock reply captured live on 2026-10-06. """ @@ -11,16 +11,25 @@ litellm_endpoint="/v1/decisions", litellm_request={ "model": "perplexity/pplx-decider-v1-27b", - "state": "Ticket (billing): The export job hangs at 99% and never finishes", - "questions": { - "defect": {"type": "noul", "instructions": "Is this a defect?"}, - "severity": { + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { "type": "choice", + "name": "severity", "instructions": "How severe is it?", - "criteria": {"low": "cosmetic", "high": "blocks users"}, + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], }, - "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, - }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], "cache": {"no-cache": True}, }, expected_provider_endpoint="/v1/decisions", @@ -60,22 +69,35 @@ }, expected_litellm_response={ "model": "pplx-decider-v1-27b", - "answers": { - "defect": {"type": "noul", "noul": 0.9989100737587077}, - "severity": { + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.9989100737587077}, + { "type": "choice", + "name": "severity", "choice": "high", + "probabilities": [ + {"value": "low", "probability": 0.0017684392321610232}, + {"value": "high", "probability": 0.9982315607678389}, + ], "confidence": 0.9964631215356778, - "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, }, - "confidence": { + { "type": "score", + "name": "confidence", "score": 0.07367392327139817, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.9263260767286018}, + {"value": 1, "label": "sure", "probability": 0.07367392327139817}, + ], "confidence": 0.8526521534572037, - "legend": {"0": "unsure", "1": "sure"}, - "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, }, + ], + "usage": { + "input_tokens": 318, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 321, }, - "usage": {"input_tokens": 318, "output_tokens": 3}, }, ) diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py index 97afcdefd86d..d9e5981ef4e6 100644 --- a/tests/integration/translation/decisions/bases/strands_decider.py +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -2,7 +2,7 @@ from integration.translation.case import TranslationTestCase -"""Strands Decider: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. +"""Strands Decider: LiteLLM speaks the OpenAI Decisions shape; the provider speaks Jev / System One. Provider side from https://github.com/strands-agents/decider (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-06. """ @@ -11,8 +11,8 @@ litellm_endpoint="/v1/decisions", litellm_request={ "model": "strands_decider/strands-decider-2B-hobson-v19", - "state": "Help! My payouts have been failing for 3 days!", - "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + "input": "Help! My payouts have been failing for 3 days!", + "questions": [{"type": "predicate", "name": "is_urgent", "instructions": "Does this convey urgency?"}], "cache": {"no-cache": True}, }, expected_provider_endpoint="/v1/systemone", @@ -30,8 +30,13 @@ }, expected_litellm_response={ "model": "strands-decider-2B-hobson-v19", - "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, - "usage": {"input_tokens": 86, "output_tokens": 1}, - "latency_ms": 140.03, + "answers": [{"type": "predicate", "name": "is_urgent", "probability": 0.8277}], + "usage": { + "input_tokens": 86, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 87, + }, }, ) diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py index acada7702c30..fe87d8c47543 100644 --- a/tests/integration/translation/decisions/bases/typesafe.py +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -2,7 +2,7 @@ from integration.translation.case import TranslationTestCase -"""TypeSafe: LiteLLM's /v1/decisions and the provider both speak Jev / System One, so the body passes through. +"""TypeSafe: LiteLLM speaks the OpenAI Decisions shape; the provider speaks Jev / System One. Provider side from https://docs.typesafe.ai (POST /v1/systemone). Mock reply captured live on 2026-10-06. """ @@ -11,16 +11,25 @@ litellm_endpoint="/v1/decisions", litellm_request={ "model": "typesafe/jev-1.13.0", - "state": "Ticket (billing): The export job hangs at 99% and never finishes", - "questions": { - "defect": {"type": "noul", "instructions": "Is this a defect?"}, - "severity": { + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { "type": "choice", + "name": "severity", "instructions": "How severe is it?", - "criteria": {"low": "cosmetic", "high": "blocks users"}, + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], }, - "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, - }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], "cache": {"no-cache": True}, }, expected_provider_endpoint="/v1/systemone", @@ -60,22 +69,32 @@ }, expected_litellm_response={ "model": "jev-1.13.0", - "answers": { - "defect": {"type": "noul", "noul": 0.78}, - "severity": { + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.78}, + { "type": "choice", + "name": "severity", "choice": "high", + "probabilities": [{"value": "low", "probability": 0.01}, {"value": "high", "probability": 0.99}], "confidence": 0.99, - "probabilities": {"low": 0.01, "high": 0.99}, }, - "confidence": { + { "type": "score", + "name": "confidence", "score": 0.51, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.49}, + {"value": 1, "label": "sure", "probability": 0.51}, + ], "confidence": 0.03, - "legend": {"0": "unsure", "1": "sure"}, - "probabilities": {"0": 0.49, "1": 0.51}, }, + ], + "usage": { + "input_tokens": 377, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 62, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 439, }, - "usage": {"input_tokens": 377, "output_tokens": 62}, }, ) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 5994bd416625..24c08081661a 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -2,8 +2,7 @@ import asyncio import json -from collections.abc import Mapping -from types import MappingProxyType +from collections.abc import Mapping, Sequence from typing import Final import pytest @@ -14,22 +13,41 @@ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.decisions import ( ChoiceAnswer, + DecisionInputMessage, DecisionsResponse, DecisionsUsage, - NoulAnswer, + PredicateAnswer, ScoreAnswer, ) -_QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( +_INPUT: Final = "The export job hangs at 99% and never finishes" +_QUESTIONS: Final[Sequence[Mapping[str, object]]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, { - "is_defect": {"type": "noul", "instructions": "Is this a defect?", "provider_field": "kept"}, - "sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}}, - "severity": {"type": "score", "criteria": ["none", "low", "high"]}, - } + "type": "choice", + "name": "sentiment", + "instructions": "How does the customer feel?", + "choices": [{"value": "positive"}, {"value": "negative", "description": "unhappy"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is it?", + "levels": [{"label": "none"}, {"label": "low"}, {"label": "high", "description": "blocks users"}], + }, ) +_SYSTEM_ONE_QUESTIONS: Final[Mapping[str, object]] = { + "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, + "sentiment": { + "type": "choice", + "instructions": "How does the customer feel?", + "criteria": {"positive": None, "negative": "unhappy"}, + }, + "severity": {"type": "score", "instructions": "How severe is it?", "criteria": ["none", "low", "blocks users"]}, +} _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 -_RESPONSE: Final[Mapping[str, object]] = { +_SYSTEM_ONE_RESPONSE: Final[Mapping[str, object]] = { "model": "jev-1.13", "answers": { "is_defect": {"type": "noul", "noul": 0.9}, @@ -49,34 +67,38 @@ }, "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, } -_STRANDS_RESPONSE: Final[Mapping[str, object]] = { - "model": "strands-decider-2B-hobson-v19", - "answers": { - "severity": { - "type": "score", - "score": 1, - "confidence": 0.7, - "legend": {"0": "none", "1": "low", "2": "high"}, - "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, - } +_EXPECTED_ANSWERS: Final[Sequence[Mapping[str, object]]] = ( + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + { + "type": "choice", + "name": "sentiment", + "choice": "positive", + "probabilities": [{"value": "positive", "probability": 0.8}, {"value": "negative", "probability": 0.2}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 1.0, + "probabilities": [ + {"value": 0, "label": "none", "probability": 0.1}, + {"value": 1, "label": "low", "probability": 0.8}, + {"value": 2, "label": "high", "probability": 0.1}, + ], + "confidence": 0.7, }, - "usage": {"input_tokens": 216, "output_tokens": 3}, - "latency_ms": 3722.17, +) +_EXPECTED_USAGE: Final[Mapping[str, object]] = { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, } _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( - ( - "perplexity", - "perplexity/pplx-decider-v1-27b", - "https://api.perplexity.ai/v1/decisions", - "pplx-decider-v1-27b", - ), + ("perplexity", "perplexity/pplx-decider-v1-27b", "https://api.perplexity.ai/v1/decisions", "pplx-decider-v1-27b"), ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), - ( - "openrouter", - "openrouter/typesafe/jev-1.13", - "https://openrouter.ai/api/alpha/decisions", - "typesafe/jev-1.13", - ), + ("openrouter", "openrouter/typesafe/jev-1.13", "https://openrouter.ai/api/alpha/decisions", "typesafe/jev-1.13"), ) @@ -103,6 +125,10 @@ async def _drain_logging_worker() -> None: await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) +def _predicate(name: str = "is_defect") -> list[Mapping[str, object]]: + return [{"type": "predicate", "name": name, "instructions": "Is this a defect?"}] + + @pytest.fixture(autouse=True) def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -111,18 +137,18 @@ def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.asyncio @pytest.mark.parametrize(("provider", "model", "url", "upstream_model"), _PROVIDERS) -async def test_adecisions_sends_the_provider_wire_contract( +async def test_adecisions_translates_to_the_system_one_wire_contract( provider: str, model: str, url: str, upstream_model: str, respx_mock: respx.MockRouter, ) -> None: - route: Final = respx_mock.post(url).respond(json=_RESPONSE) + route: Final = respx_mock.post(url).respond(json=_SYSTEM_ONE_RESPONSE) response: Final = await litellm.adecisions( model=model, - state={"source": "unit-test"}, + input=_INPUT, questions=_QUESTIONS, api_key="caller-key", extra_headers={ @@ -141,21 +167,126 @@ async def test_adecisions_sends_the_provider_wire_contract( assert request.headers["x-request-tag"] == "decisions-test" assert json.loads(request.content) == { "model": upstream_model, - "state": {"source": "unit-test"}, + "state": _INPUT, + "questions": _SYSTEM_ONE_QUESTIONS, + } + assert response.model_dump(mode="json") == { + "model": "jev-1.13", + "answers": list(_EXPECTED_ANSWERS), + "usage": _EXPECTED_USAGE, + } + assert isinstance(response.answers[0], PredicateAnswer) + assert isinstance(response.answers[1], ChoiceAnswer) + assert isinstance(response.answers[2], ScoreAnswer) + assert response._hidden_params["custom_llm_provider"] == provider + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("label", "input_value", "questions"), + ( + ( + "input_image", + [{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]}], + _predicate(), + ), + ("boolean choice", _INPUT, [{"type": "choice", "instructions": "Refund?", "choices": [{"value": True}]}]), + ("unique name", _INPUT, [*_predicate(), *_predicate()]), + ( + "repeated choice", + _INPUT, + [{"type": "choice", "instructions": "Refund?", "choices": [{"value": "yes"}, {"value": "yes"}]}], + ), + ), +) +async def test_system_one_providers_refuse_what_they_cannot_express_before_http( + label: str, + input_value: object, + questions: Sequence[Mapping[str, object]], + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.BadRequestError, match=label): + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", input=input_value, questions=questions, api_key="caller-key" + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_positional_system_one_keys_never_shadow_a_supplied_name(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={ + "model": "pplx-decider-v1-27b", + "answers": {"_q0": {"type": "noul", "noul": 0.25}, "q0": {"type": "noul", "noul": 0.75}}, + "usage": {"input_tokens": 10, "output_tokens": 2}, + } + ) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + input=_INPUT, + questions=[{"type": "predicate", "instructions": "Is this a defect?"}, *_predicate("q0")], + api_key="caller-key", + ) + + assert route.called + assert tuple(json.loads(respx_mock.calls[0].request.content)["questions"]) == ("_q0", "q0") + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": None, "probability": 0.25}, + {"type": "predicate", "name": "q0", "probability": 0.75}, + ] + + +@pytest.mark.asyncio +async def test_unnamed_questions_get_positional_system_one_keys(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={ + "model": "pplx-decider-v1-27b", + "answers": {"q0": {"type": "noul", "noul": 0.25}, "q1": {"type": "noul", "noul": 0.75}}, + "usage": {"input_tokens": 10, "output_tokens": 2}, + } + ) + user_messages: Final = [ + {"role": "user", "content": "first"}, + {"role": "user", "content": [{"type": "input_text", "text": "second"}]}, + ] + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + input=user_messages, + questions=[ + {"type": "predicate", "instructions": "Is this a defect?"}, + {"type": "predicate", "instructions": "Is this urgent?"}, + ], + api_key="caller-key", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": "first\nsecond", "questions": { - "is_defect": { - "type": "noul", - "instructions": "Is this a defect?", - "provider_field": "kept", - }, - "sentiment": {"type": "choice", "criteria": {"positive": None, "negative": "unhappy"}}, - "severity": {"type": "score", "criteria": ["none", "low", "high"]}, + "q0": {"type": "noul", "instructions": "Is this a defect?"}, + "q1": {"type": "noul", "instructions": "Is this urgent?"}, }, } - assert isinstance(response.answers["is_defect"], NoulAnswer) - assert isinstance(response.answers["sentiment"], ChoiceAnswer) - assert isinstance(response.answers["severity"], ScoreAnswer) - assert response._hidden_params["custom_llm_provider"] == provider + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": None, "probability": 0.25}, + {"type": "predicate", "name": None, "probability": 0.75}, + ] + + +@pytest.mark.asyncio +async def test_system_one_reply_missing_an_answer_is_a_server_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={"model": "pplx-decider-v1-27b", "answers": {}, "usage": {"input_tokens": 10, "output_tokens": 0}} + ) + + with pytest.raises(litellm.InternalServerError, match="no predicate answer for question 'is_defect'"): + await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", input=_INPUT, questions=_predicate(), api_key="caller-key" + ) @pytest.mark.asyncio @@ -165,91 +296,94 @@ async def test_router_dispatches_typesafe_decisions_without_api_base( ) -> None: monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) monkeypatch.delenv("TYPESAFE_API_BASE", raising=False) - provider_resolution: Final = litellm.get_llm_provider("typesafe/jev-latest") + assert litellm.get_llm_provider("typesafe/jev-latest")[:2] == ("jev-latest", "typesafe") + router: Final = litellm.Router( + model_list=[{"model_name": "jev", "litellm_params": {"model": "typesafe/jev-latest", "api_key": "k"}}] + ) + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_SYSTEM_ONE_RESPONSE) + + response: Final = await router.adecisions(model="jev", input=_INPUT, questions=_QUESTIONS) + + assert upstream.called + assert len(respx_mock.calls) == 1 + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "jev-latest", + "state": _INPUT, + "questions": _SYSTEM_ONE_QUESTIONS, + } + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer k" + sentiment: Final = response.answers[1] + assert isinstance(sentiment, ChoiceAnswer) + assert sentiment.choice == "positive" - assert provider_resolution[:2] == ("jev-latest", "typesafe") +@pytest.mark.asyncio +async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_already_resolved( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) router: Final = litellm.Router( model_list=[ { "model_name": "jev", "litellm_params": { - "model": "typesafe/jev-latest", - "api_key": "k", + "model": "openrouter/typesafe/jev-1.13", + "api_key": "deployment-key", + "api_base": "https://egress.example/openrouter", }, } ] ) - upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) - - response: Final = await router.adecisions( - model="jev", - state="router-test", - questions={ - "sentiment": { - "type": "choice", - "criteria": {"positive": None, "negative": "unhappy"}, - } - }, + upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond( + json=_SYSTEM_ONE_RESPONSE ) + await router.adecisions(model="jev", input=_INPUT, questions=_predicate()) + assert upstream.called - assert len(respx_mock.calls) == 1 - assert json.loads(respx_mock.calls[0].request.content) == { - "model": "jev-latest", - "state": "router-test", - "questions": { - "sentiment": { - "type": "choice", - "criteria": {"positive": None, "negative": "unhappy"}, - } - }, - } - assert respx_mock.calls[0].request.headers["authorization"] == "Bearer k" - assert isinstance(response.answers["sentiment"], ChoiceAnswer) - assert response.answers["sentiment"].choice == "positive" + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(respx_mock.calls[0].request.content)["model"] == "typesafe/jev-1.13" def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None: - route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_SYSTEM_ONE_RESPONSE) response: Final = litellm.decisions( - model="perplexity/pplx-decider-v1-27b", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - api_key="caller-key", + model="perplexity/pplx-decider-v1-27b", input=_INPUT, questions=_QUESTIONS, api_key="caller-key" ) assert route.called - assert response.model == "jev-1.13" + assert json.loads(respx_mock.calls[0].request.content)["questions"] == _SYSTEM_ONE_QUESTIONS + assert response.model_dump(mode="json") == { + "model": "jev-1.13", + "answers": list(_EXPECTED_ANSWERS), + "usage": _EXPECTED_USAGE, + } -def test_openrouter_response_keeps_provider_fields(respx_mock: respx.MockRouter) -> None: +def test_system_one_provider_extras_are_not_part_of_the_response(respx_mock: respx.MockRouter) -> None: payload: Final = { - **_RESPONSE, + **_SYSTEM_ONE_RESPONSE, "id": "decision-1", "provider": "typesafe", - "usage": {**_RESPONSE["usage"], "cost": 0.25}, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS, "cost": 0.25}, } respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=payload) response: Final = litellm.decisions( - model="openrouter/typesafe/jev-1.13", - state="review", - questions=_QUESTIONS, - api_key="caller-key", + model="openrouter/typesafe/jev-1.13", input=_INPUT, questions=_QUESTIONS, api_key="caller-key" ) - assert response.model_extra["id"] == "decision-1" - assert response.model_extra["provider"] == "typesafe" + assert response.model_extra == {} assert response.usage is not None - assert response.usage.model_extra["cost"] == 0.25 + assert response.usage.model_dump(mode="json") == _EXPECTED_USAGE def test_decisions_cost_uses_litellm_token_pricing() -> None: response: Final = DecisionsResponse( model="pplx-decider-v1-27b", - answers={}, + answers=[], usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), ) response.set_hidden_params({"model": "perplexity/pplx-decider-v1-27b", "custom_llm_provider": "perplexity"}) @@ -265,7 +399,7 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: def test_decisions_response_hidden_params_getter_preserves_mutable_identity() -> None: - response: Final = DecisionsResponse(model="decider", answers={}, usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=None) assert response.hidden_params is response._hidden_params @@ -275,17 +409,14 @@ def test_decisions_response_hidden_params_getter_preserves_mutable_identity() -> @pytest.mark.asyncio async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.MockRouter) -> None: - respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_SYSTEM_ONE_RESPONSE) recording_logger: Final = _RecordingLogger() original_callbacks: Final = litellm.callbacks litellm.callbacks = [recording_logger] try: await litellm.adecisions( - model="perplexity/pplx-decider-v1-27b", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - api_key="caller-key", + model="perplexity/pplx-decider-v1-27b", input=_INPUT, questions=_predicate(), api_key="caller-key" ) await _drain_logging_worker() finally: @@ -301,17 +432,37 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo assert recording_logger.standard_logging_object["response_cost"] == pytest.approx(expected_cost) assert recording_logger.standard_logging_object["prompt_tokens"] == _INPUT_TOKENS assert recording_logger.standard_logging_object["completion_tokens"] == _OUTPUT_TOKENS + assert recording_logger.standard_logging_object["messages"] == [{"role": "user", "content": _INPUT}] @pytest.mark.asyncio -async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): +async def test_decisions_log_typed_input_messages(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_SYSTEM_ONE_RESPONSE) + recording_logger: Final = _RecordingLogger() + original_callbacks: Final = litellm.callbacks + litellm.callbacks = [recording_logger] + + try: await litellm.adecisions( - model="unknown/jev-1.13", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + model="perplexity/pplx-decider-v1-27b", + input=[DecisionInputMessage(role="user", content=_INPUT)], + questions=_predicate(), api_key="caller-key", ) + await _drain_logging_worker() + finally: + litellm.callbacks = original_callbacks + + assert route.called + assert recording_logger.standard_logging_object is not None + logged_messages: Final = recording_logger.standard_logging_object["messages"] + assert json.loads(logged_messages[0]["content"]) == [{"role": "user", "content": _INPUT}] + + +@pytest.mark.asyncio +async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): + await litellm.adecisions(model="unknown/jev-1.13", input=_INPUT, questions=_predicate(), api_key="caller-key") assert len(respx_mock.calls) == 0 @@ -320,10 +471,7 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"): await litellm.adecisions( - model="anthropic/claude-sonnet-4-5", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - api_key="caller-key", + model="anthropic/claude-sonnet-4-5", input=_INPUT, questions=_predicate(), api_key="caller-key" ) assert len(respx_mock.calls) == 0 @@ -331,12 +479,12 @@ async def test_provider_without_decisions_support_is_rejected_before_http(respx_ @pytest.mark.asyncio async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None: - route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_SYSTEM_ONE_RESPONSE) response: Final = await litellm.adecisions( model="perplexity/pplx-decider-v1-27b", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + input=_INPUT, + questions=_predicate(), api_key="caller-key", custom_llm_provider="", ) @@ -347,59 +495,20 @@ async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: @pytest.mark.asyncio -async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: respx.MockRouter) -> None: - respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( - json={"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}} - ) - - with pytest.raises(litellm.InternalServerError, match="unexpected response"): - await litellm.adecisions( - model="perplexity/pplx-decider-v1-27b", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - api_key="caller-key", - ) - - -@pytest.mark.asyncio -async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_already_resolved( - monkeypatch: pytest.MonkeyPatch, - respx_mock: respx.MockRouter, -) -> None: - monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) - router: Final = litellm.Router( - model_list=[ - { - "model_name": "jev", - "litellm_params": { - "model": "openrouter/typesafe/jev-1.13", - "api_key": "deployment-key", - "api_base": "https://egress.example/openrouter", - }, - } - ] - ) - upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE) - - await router.adecisions( - model="jev", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - ) - - assert upstream.called - assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key" - assert json.loads(respx_mock.calls[0].request.content)["model"] == "typesafe/jev-1.13" - - -@pytest.mark.asyncio -async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: +@pytest.mark.parametrize( + "questions", + ( + [{"type": "choice", "name": "sentiment", "instructions": "How?"}], + [{"type": "noul", "name": "is_defect", "instructions": "Is this a defect?"}], + {"is_defect": {"type": "predicate", "instructions": "Is this a defect?"}}, + [], + ), + ids=("choice without choices", "jev question type", "questions map", "no questions"), +) +async def test_invalid_questions_are_rejected_before_http(questions: object, respx_mock: respx.MockRouter) -> None: with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): await litellm.adecisions( - model="perplexity/pplx-decider-v1-27b", - state="review", - questions={"sentiment": {"type": "choice"}}, - api_key="caller-key", + model="perplexity/pplx-decider-v1-27b", input=_INPUT, questions=questions, api_key="caller-key" ) assert len(respx_mock.calls) == 0 @@ -407,16 +516,12 @@ async def test_invalid_question_is_rejected_before_http(respx_mock: respx.MockRo def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter) -> None: respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( - status_code=400, - json={"error": {"message": "invalid decision"}}, + status_code=400, json={"error": {"message": "invalid decision"}} ) with pytest.raises(litellm.BadRequestError): litellm.decisions( - model="perplexity/pplx-decider-v1-27b", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - api_key="caller-key", + model="perplexity/pplx-decider-v1-27b", input=_INPUT, questions=_predicate(), api_key="caller-key" ) @@ -426,12 +531,12 @@ def test_server_key_is_sent_to_an_explicit_api_base( ) -> None: monkeypatch.setenv("PERPLEXITYAI_API_KEY", "server-key") monkeypatch.delenv("PERPLEXITY_API_KEY", raising=False) - route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) + route: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_SYSTEM_ONE_RESPONSE) litellm.decisions( model="perplexity/pplx-decider-v1-27b", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + input=_INPUT, + questions=_predicate(), api_base="https://egress.example/perplexity", ) @@ -452,27 +557,21 @@ async def test_cloudflare_clef_resolves_model_and_response_envelope( monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) response_body: Final[Mapping[str, object]] = ( - {"result": _RESPONSE, "success": True, "errors": [], "messages": []} if wrapped else _RESPONSE + {"result": _SYSTEM_ONE_RESPONSE, "success": True, "errors": [], "messages": []} + if wrapped + else _SYSTEM_ONE_RESPONSE ) route: Final = respx_mock.post( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" ).respond(json=response_body) - response: Final = await litellm.adecisions( - model=model, - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - ) + response: Final = await litellm.adecisions(model=model, input=_INPUT, questions=_QUESTIONS) assert route.called request: Final = respx_mock.calls[0].request assert request.headers["authorization"] == "Bearer cloudflare-key" - assert json.loads(request.content) == { - "model": "clef", - "state": "review", - "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - } - assert response.answers == DecisionsResponse.model_validate(_RESPONSE).answers + assert json.loads(request.content) == {"model": "clef", "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + assert [answer.model_dump(mode="json") for answer in response.answers] == list(_EXPECTED_ANSWERS) assert response._hidden_params["model"] == "cloudflare/@cf/cloudflare/clef" @@ -486,13 +585,9 @@ async def test_cloudflare_clef_flash_uses_flash_endpoint_and_request_model( monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) route: Final = respx_mock.post( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-flash" - ).respond(json=_RESPONSE) + ).respond(json=_SYSTEM_ONE_RESPONSE) - await litellm.adecisions( - model="cloudflare/clef-flash", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - ) + await litellm.adecisions(model="cloudflare/clef-flash", input=_INPUT, questions=_QUESTIONS) assert route.called assert json.loads(respx_mock.calls[0].request.content)["model"] == "clef-flash" @@ -508,13 +603,9 @@ async def test_cloudflare_api_base_from_env_uses_workers_ai_run_path( monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) route: Final = respx_mock.post( "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" - ).respond(json=_RESPONSE) + ).respond(json=_SYSTEM_ONE_RESPONSE) - await litellm.adecisions( - model="cloudflare/clef", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - ) + await litellm.adecisions(model="cloudflare/clef", input=_INPUT, questions=_QUESTIONS) assert route.called @@ -529,11 +620,7 @@ async def test_cloudflare_requires_account_id_or_api_base_before_http( monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") with pytest.raises(litellm.BadRequestError, match="Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"): - await litellm.adecisions( - model="cloudflare/clef", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - ) + await litellm.adecisions(model="cloudflare/clef", input=_INPUT, questions=_QUESTIONS) assert len(respx_mock.calls) == 0 @@ -547,14 +634,10 @@ async def test_cloudflare_clef_cost_uses_the_model_cost_map( monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef").respond( - json=_RESPONSE + json=_SYSTEM_ONE_RESPONSE ) - response: Final = await litellm.adecisions( - model="cloudflare/clef", - state="review", - questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - ) + response: Final = await litellm.adecisions(model="cloudflare/clef", input=_INPUT, questions=_QUESTIONS) cost: Final = litellm.completion_cost(completion_response=response) clef_cost: Final = litellm.model_cost["cloudflare/@cf/cloudflare/clef"] @@ -576,36 +659,36 @@ async def test_strands_decider_requires_api_base_before_http( with pytest.raises(litellm.BadRequestError, match="api_base is required"): await litellm.adecisions( - model="strands_decider/strands-decider-2B-hobson-v19", - state="review", - questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + model="strands_decider/strands-decider-2B-hobson-v19", input=_INPUT, questions=_QUESTIONS ) assert len(respx_mock.calls) == 0 @pytest.mark.asyncio -async def test_strands_decider_without_key_preserves_response_extras( +async def test_strands_decider_without_key_sends_no_authorization( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, ) -> None: monkeypatch.delenv("STRANDS_DECIDER_API_BASE", raising=False) monkeypatch.delenv("STRANDS_DECIDER_API_KEY", raising=False) - route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond( + json={**_SYSTEM_ONE_RESPONSE, "model": "strands-decider-2B-hobson-v19", "latency_ms": 140.03} + ) response: Final = await litellm.adecisions( model="strands_decider/strands-decider-2B-hobson-v19", - state="review", - questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + input=_INPUT, + questions=_QUESTIONS, api_base="https://strands.example", ) assert route.called assert "authorization" not in respx_mock.calls[0].request.headers - assert response.model_extra["latency_ms"] == _STRANDS_RESPONSE["latency_ms"] - severity: Final = response.answers["severity"] + assert response.model == "strands-decider-2B-hobson-v19" + severity: Final = response.answers[2] assert isinstance(severity, ScoreAnswer) - assert severity.legend == {"0": "none", "1": "low", "2": "high"} + assert [probability.label for probability in severity.probabilities] == ["none", "low", "high"] @pytest.mark.asyncio @@ -615,12 +698,12 @@ async def test_strands_decider_uses_key_from_matching_environment_base( ) -> None: monkeypatch.setenv("STRANDS_DECIDER_API_BASE", "https://strands.example") monkeypatch.setenv("STRANDS_DECIDER_API_KEY", "strands-key") - route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_SYSTEM_ONE_RESPONSE) await litellm.adecisions( model="strands_decider/strands-decider-2B-hobson-v19", - state="review", - questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, + input=_INPUT, + questions=_QUESTIONS, api_base="https://strands.example", ) @@ -647,14 +730,10 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( } ] ) - route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) + route: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_SYSTEM_ONE_RESPONSE) - response: Final = await router.adecisions( - model="strands", - state="review", - questions={"severity": {"type": "score", "criteria": ["none", "low", "high"]}}, - ) + response: Final = await router.adecisions(model="strands", input=_INPUT, questions=_QUESTIONS) assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider") assert route.called - assert response.model == _STRANDS_RESPONSE["model"] + assert response.model == "jev-1.13" diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 9a686ba9ab3d..a0e3c9fdd05e 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -636,8 +636,8 @@ async def test_ahealth_check_evaluation_uses_configured_probe_state_and_question { "model": "perplexity/pplx-decider-v1-27b", "api_key": "sk-test", - "state": "custom probe", - "questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}}, + "input": "custom probe", + "questions": [{"type": "predicate", "name": "ok", "instructions": "Is it ok?"}], }, mode=None, ) diff --git a/tests/unit/litellm_core_utils/test_hidden_params.py b/tests/unit/litellm_core_utils/test_hidden_params.py index 9d1e1e8dcebc..ad5c6e3a9ebb 100644 --- a/tests/unit/litellm_core_utils/test_hidden_params.py +++ b/tests/unit/litellm_core_utils/test_hidden_params.py @@ -189,8 +189,8 @@ class PlainResponse: fallback_response: Final = PlainResponse() setattr(fallback_response, HIDDEN_PARAMS_ATTR, HiddenParams()) fallback_view: Final = get_or_create_hidden_params(fallback_response) - merged_after_fallback: Final = {**fallback_view, **{"model_id": "m1"}} - merged_before_fallback: Final = {**{"model_id": "m1"}, **fallback_view} + merged_after_fallback: Final = {**fallback_view, "model_id": "m1"} + merged_before_fallback: Final = {"model_id": "m1", **fallback_view} assert merged_after_fallback == {"model_id": "m1"} assert merged_before_fallback == {"model_id": "m1"} @@ -243,7 +243,7 @@ def __init__(self) -> None: def test_set_hidden_params_replaces_frozen_decisions_response_private_attr() -> None: - response: Final = DecisionsResponse(model="decider", answers={}, usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=None) replacement: Final = {"replacement": True} set_hidden_params(response, replacement) diff --git a/tests/unit/llms/base_llm/decisions/test_systemone.py b/tests/unit/llms/base_llm/decisions/test_systemone.py index f1c7177e83c7..937e18b565d8 100644 --- a/tests/unit/llms/base_llm/decisions/test_systemone.py +++ b/tests/unit/llms/base_llm/decisions/test_systemone.py @@ -13,7 +13,7 @@ question_keys, system_one_request, ) -from litellm.types.openai_decisions import ChoiceAnswer, DecisionsRequest, PredicateAnswer, ScoreAnswer +from litellm.types.decisions import ChoiceAnswer, DecisionsRequest, PredicateAnswer, ScoreAnswer _INPUT: Final = "The export job hangs at 99% and never finishes" _QUESTIONS: Final[Sequence[Mapping[str, object]]] = ( diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 6b1ac9e3404e..5678f3c6a55d 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -43,8 +43,8 @@ } _REQUEST: Final[Mapping[str, object]] = { "model": "decider", - "state": {"source": "proxy-test"}, - "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "input": "proxy-test", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], } @@ -88,7 +88,7 @@ def test_proxy_decisions_route_returns_answers_and_cost( response: Final = client.post(endpoint, json=_REQUEST) assert response.status_code == 200, response.text - assert response.json()["answers"] == _RESPONSE["answers"] + assert response.json()["answers"] == [{"type": "predicate", "name": "is_defect", "probability": 0.9}] assert "_hidden_params" not in response.json() perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] expected_cost: Final = _INPUT_TOKENS * float(perplexity_cost["input_cost_per_token"]) + _OUTPUT_TOKENS * float( @@ -100,7 +100,7 @@ def test_proxy_decisions_route_returns_answers_and_cost( assert upstream.called assert json.loads(upstream.calls[0].request.content) == { "model": "pplx-decider-v1-27b", - "state": {"source": "proxy-test"}, + "state": "proxy-test", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key" @@ -129,17 +129,17 @@ def test_proxy_decisions_dispatches_typesafe_deployment( "/v1/decisions", json={ "model": "jev", - "state": {"source": "proxy-test"}, - "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "input": "proxy-test", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], }, ) assert response.status_code == 200, response.text - assert response.json()["answers"] == _RESPONSE["answers"] + assert response.json()["answers"] == [{"type": "predicate", "name": "is_defect", "probability": 0.9}] assert upstream.called assert json.loads(upstream.calls[0].request.content) == { "model": "jev-latest", - "state": {"source": "proxy-test"}, + "state": "proxy-test", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } @@ -180,8 +180,8 @@ def test_proxy_decisions_unknown_model_is_a_client_error( "/v1/decisions", json={ "model": "missing-model", - "state": "review", - "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], }, ) @@ -194,14 +194,14 @@ def test_proxy_decisions_unknown_model_is_a_client_error( ( { "model": "decider", - "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], }, { "model": "decider", - "state": {"source": "proxy-test"}, + "input": "proxy-test", }, ), - ids=("missing_state", "missing_questions"), + ids=("missing_input", "missing_questions"), ) def test_proxy_decisions_missing_required_field_is_a_client_error( client: TestClient, @@ -241,17 +241,17 @@ def test_proxy_decisions_dispatches_strands_decider( "/v1/decisions", json={ "model": "strands", - "state": {"source": "proxy-test"}, - "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + "input": "proxy-test", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], }, ) assert response.status_code == 200, response.text - assert response.json()["answers"] == _STRANDS_RESPONSE["answers"] + assert response.json()["answers"] == [{"type": "predicate", "name": "is_defect", "probability": 0.9}] assert upstream.called assert json.loads(upstream.calls[0].request.content) == { "model": "strands-decider-2B-hobson-v19", - "state": {"source": "proxy-test"}, + "state": "proxy-test", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } assert "authorization" not in upstream.calls[0].request.headers @@ -270,7 +270,7 @@ def test_proxy_decisions_without_model_uses_the_proxy_default_model( ) assert response.status_code == 200, response.text - assert response.json()["answers"] == _RESPONSE["answers"] + assert response.json()["answers"] == [{"type": "predicate", "name": "is_defect", "probability": 0.9}] assert upstream.called assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b" diff --git a/tests/unit/router_utils/test_add_retry_fallback_headers.py b/tests/unit/router_utils/test_add_retry_fallback_headers.py index 1f4ac3be6ac3..971a3f8bb695 100644 --- a/tests/unit/router_utils/test_add_retry_fallback_headers.py +++ b/tests/unit/router_utils/test_add_retry_fallback_headers.py @@ -259,7 +259,7 @@ def __init__(self): def test_add_fallback_headers_to_frozen_decisions_response() -> None: - response: Final = DecisionsResponse(model="decider", answers={}, usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=None) result: Final = add_fallback_headers_to_response(response=response, attempted_fallbacks=1) @@ -268,7 +268,7 @@ def test_add_fallback_headers_to_frozen_decisions_response() -> None: def test_ensure_response_additional_headers_updates_frozen_decisions_response() -> None: - response: Final = DecisionsResponse(model="decider", answers={}, usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=None) additional_headers: Final = ensure_response_additional_headers(response) diff --git a/tests/unit/types/test_openai_decisions.py b/tests/unit/types/test_decisions.py similarity index 98% rename from tests/unit/types/test_openai_decisions.py rename to tests/unit/types/test_decisions.py index 4cbd4212dcb7..255d7358ef9b 100644 --- a/tests/unit/types/test_openai_decisions.py +++ b/tests/unit/types/test_decisions.py @@ -6,7 +6,7 @@ import pytest from pydantic import TypeAdapter, ValidationError -from litellm.types.openai_decisions import ( +from litellm.types.decisions import ( ChoiceAnswer, DecisionsRequestBody, DecisionsResponse, From 9b8882dcb93bdf0c982ae1270bf57a9ba380f694 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 20:59:06 +0000 Subject: [PATCH 07/15] test(decisions): build the usage the OpenAI response schema now requires Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/decisions/test_main.py | 20 ++++++++-- .../litellm_core_utils/test_hidden_params.py | 17 ++++++++- .../test_add_retry_fallback_headers.py | 38 ++++++++++--------- 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 24c08081661a..3ba4126667fd 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -14,6 +14,8 @@ from litellm.types.decisions import ( ChoiceAnswer, DecisionInputMessage, + DecisionsInputTokensDetails, + DecisionsOutputTokensDetails, DecisionsResponse, DecisionsUsage, PredicateAnswer, @@ -95,6 +97,13 @@ "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, } +_ZERO_USAGE: Final = DecisionsUsage( + input_tokens=0, + input_tokens_details=DecisionsInputTokensDetails(cached_tokens=0, cache_write_tokens=0), + output_tokens=0, + output_tokens_details=DecisionsOutputTokensDetails(reasoning_tokens=0), + total_tokens=0, +) _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ("perplexity", "perplexity/pplx-decider-v1-27b", "https://api.perplexity.ai/v1/decisions", "pplx-decider-v1-27b"), ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), @@ -376,7 +385,6 @@ def test_system_one_provider_extras_are_not_part_of_the_response(respx_mock: res ) assert response.model_extra == {} - assert response.usage is not None assert response.usage.model_dump(mode="json") == _EXPECTED_USAGE @@ -384,7 +392,13 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: response: Final = DecisionsResponse( model="pplx-decider-v1-27b", answers=[], - usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), + usage=DecisionsUsage( + input_tokens=_INPUT_TOKENS, + input_tokens_details=DecisionsInputTokensDetails(cached_tokens=0, cache_write_tokens=0), + output_tokens=_OUTPUT_TOKENS, + output_tokens_details=DecisionsOutputTokensDetails(reasoning_tokens=0), + total_tokens=_INPUT_TOKENS + _OUTPUT_TOKENS, + ), ) response.set_hidden_params({"model": "perplexity/pplx-decider-v1-27b", "custom_llm_provider": "perplexity"}) @@ -399,7 +413,7 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: def test_decisions_response_hidden_params_getter_preserves_mutable_identity() -> None: - response: Final = DecisionsResponse(model="decider", answers=(), usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=_ZERO_USAGE) assert response.hidden_params is response._hidden_params diff --git a/tests/unit/litellm_core_utils/test_hidden_params.py b/tests/unit/litellm_core_utils/test_hidden_params.py index ad5c6e3a9ebb..194ec9124e41 100644 --- a/tests/unit/litellm_core_utils/test_hidden_params.py +++ b/tests/unit/litellm_core_utils/test_hidden_params.py @@ -11,10 +11,23 @@ set_hidden_params, ) from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig -from litellm.types.decisions import DecisionsResponse +from litellm.types.decisions import ( + DecisionsInputTokensDetails, + DecisionsOutputTokensDetails, + DecisionsResponse, + DecisionsUsage, +) from litellm.types.llms.base import HiddenParams from litellm.types.utils import ModelResponse, TextChoices, TextCompletionResponse, Usage +_ZERO_USAGE: Final = DecisionsUsage( + input_tokens=0, + input_tokens_details=DecisionsInputTokensDetails(cached_tokens=0, cache_write_tokens=0), + output_tokens=0, + output_tokens_details=DecisionsOutputTokensDetails(reasoning_tokens=0), + total_tokens=0, +) + def test_get_and_set_hidden_params_on_plain_object() -> None: class PlainResponse: @@ -243,7 +256,7 @@ def __init__(self) -> None: def test_set_hidden_params_replaces_frozen_decisions_response_private_attr() -> None: - response: Final = DecisionsResponse(model="decider", answers=(), usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=_ZERO_USAGE) replacement: Final = {"replacement": True} set_hidden_params(response, replacement) diff --git a/tests/unit/router_utils/test_add_retry_fallback_headers.py b/tests/unit/router_utils/test_add_retry_fallback_headers.py index 971a3f8bb695..752cecdb22cb 100644 --- a/tests/unit/router_utils/test_add_retry_fallback_headers.py +++ b/tests/unit/router_utils/test_add_retry_fallback_headers.py @@ -14,7 +14,20 @@ get_hidden_params_dict, replace_complexity_router_headers, ) -from litellm.types.decisions import DecisionsResponse +from litellm.types.decisions import ( + DecisionsInputTokensDetails, + DecisionsOutputTokensDetails, + DecisionsResponse, + DecisionsUsage, +) + +_ZERO_USAGE: Final = DecisionsUsage( + input_tokens=0, + input_tokens_details=DecisionsInputTokensDetails(cached_tokens=0, cache_write_tokens=0), + output_tokens=0, + output_tokens_details=DecisionsOutputTokensDetails(reasoning_tokens=0), + total_tokens=0, +) class StreamingWrapper: @@ -197,15 +210,8 @@ def test_add_fallback_headers_serializes_fallback_errors(): ) assert result is response - assert response._hidden_params["additional_headers"][ - "x-litellm-attempted-fallbacks" - ] == 1 - assert ( - json.loads( - response._hidden_params["additional_headers"]["x-litellm-fallback-errors"] - ) - == fallback_errors - ) + assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + assert json.loads(response._hidden_params["additional_headers"]["x-litellm-fallback-errors"]) == fallback_errors def test_add_retry_headers_to_streaming_wrapper(): @@ -231,9 +237,7 @@ class InnerHiddenParams(BaseModel): class Response: def __init__(self): - self._hidden_params = InnerHiddenParams( - additional_headers={"x-custom": "value"} - ) + self._hidden_params = InnerHiddenParams(additional_headers={"x-custom": "value"}) result = get_hidden_params_dict(Response()) assert result == {"additional_headers": {"x-custom": "value"}} @@ -259,7 +263,7 @@ def __init__(self): def test_add_fallback_headers_to_frozen_decisions_response() -> None: - response: Final = DecisionsResponse(model="decider", answers=(), usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=_ZERO_USAGE) result: Final = add_fallback_headers_to_response(response=response, attempted_fallbacks=1) @@ -268,7 +272,7 @@ def test_add_fallback_headers_to_frozen_decisions_response() -> None: def test_ensure_response_additional_headers_updates_frozen_decisions_response() -> None: - response: Final = DecisionsResponse(model="decider", answers=(), usage=None) + response: Final = DecisionsResponse(model="decider", answers=(), usage=_ZERO_USAGE) additional_headers: Final = ensure_response_additional_headers(response) @@ -321,9 +325,7 @@ def test_get_fallback_errors_from_headers_existing_list_passthrough(): def test_get_fallback_errors_from_headers_invalid_json_returns_empty(): - result = get_fallback_errors_from_headers( - {"x-litellm-fallback-errors": "not-valid-json-{"} - ) + result = get_fallback_errors_from_headers({"x-litellm-fallback-errors": "not-valid-json-{"}) assert result == [] From 353e4fcb0528289e8362e1ebaba1c89481ac7a98 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 21:16:49 +0000 Subject: [PATCH 08/15] fix(lens): ask signal questions through the OpenAI Decisions schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/lens/signals.py | 43 ++++----- litellm/proxy/proxy_server.py | 5 +- tests/unit/proxy/lens/test_signals.py | 131 ++++++++++++++------------ 3 files changed, 95 insertions(+), 84 deletions(-) diff --git a/litellm/proxy/lens/signals.py b/litellm/proxy/lens/signals.py index cfe273eb6af8..c614c27b55f4 100644 --- a/litellm/proxy/lens/signals.py +++ b/litellm/proxy/lens/signals.py @@ -1,7 +1,7 @@ import asyncio import hashlib import json -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timedelta, timezone from itertools import accumulate from types import MappingProxyType @@ -154,21 +154,22 @@ def normalize_database_timestamp(cls, value: datetime | None) -> datetime | None return value -class NoulAnswer(Record): +class PredicateAnswer(Record): model_config = ConfigDict(extra="ignore", allow_inf_nan=False, from_attributes=True) - type: Literal["noul"] - noul: float = Field(ge=0, le=1, allow_inf_nan=False) + type: Literal["predicate"] + name: str + probability: float = Field(ge=0, le=1, allow_inf_nan=False) class DecisionsOutput(Record): model_config = ConfigDict(extra="ignore", from_attributes=True) - answers: Mapping[str, object] + answers: Sequence[object] DecisionState: TypeAlias = Mapping[str, object] -DecisionQuestions: TypeAlias = Mapping[str, Mapping[str, str]] +DecisionQuestions: TypeAlias = Sequence[Mapping[str, str]] Clock: TypeAlias = Callable[[], datetime] RouterReady: TypeAlias = Callable[[], bool] @@ -178,7 +179,7 @@ async def __call__( self, *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], @@ -322,13 +323,19 @@ async def signal_state(reader: SourceReader, scope: Scope, execution: Execution) } -def _noul_score(value: object) -> float | None: +def _predicate_answer(value: object) -> PredicateAnswer | None: try: - return NoulAnswer.model_validate(value).noul + return PredicateAnswer.model_validate(value) except ValidationError: return None +def _predicate_scores(answers: Sequence[object], config: SignalConfig) -> Mapping[str, float]: + wanted: Final = frozenset(signal.id for signal in config.signals) + parsed: Final = tuple(answer for value in answers if (answer := _predicate_answer(value)) is not None) + return MappingProxyType({answer.name: answer.probability for answer in parsed if answer.name in wanted}) + + class SignalClassifier: def __init__(self, reader: SourceReader, completion: DecisionsCall, clock: Clock) -> None: self.reader: Final = reader @@ -338,31 +345,25 @@ def __init__(self, reader: SourceReader, completion: DecisionsCall, clock: Clock async def classify(self, scope: Scope, execution: Execution, config: SignalConfig) -> SignalAttempt: try: state: Final = await signal_state(self.reader, scope, execution) - questions: Final = { - signal.id: {"type": "noul", "instructions": signal.question} for signal in config.signals - } + questions: Final = tuple( + {"type": "predicate", "name": signal.id, "instructions": signal.question} for signal in config.signals + ) with lens_analysis(), inherit_message_logging_privacy(True): response: Final = await self.completion( model=config.model, - state=state, + input=json.dumps(state, sort_keys=True), questions=questions, timeout=60, metadata={"tags": ["litellm-lens-signals"]}, ) output: Final = DecisionsOutput.model_validate(response) - scores: Final = MappingProxyType( - { - signal.id: score - for signal in config.signals - if (score := _noul_score(output.answers.get(signal.id))) is not None - } - ) + scores: Final = _predicate_scores(output.answers, config) if len(scores) != len(config.signals): return SignalAttempt( status="failed", scores=scores, model=config.model, - error="Decisions response omitted a configured noul answer", + error="Decisions response omitted a configured predicate answer", ) return SignalAttempt(status="classified", scores=scores, model=config.model) except Exception as error: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 68e10467c63f..f9d890c735aa 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -586,7 +586,6 @@ def generate_feedback_box(): from litellm.proxy.lens.signals import ( DecisionQuestions, DecisionsCall, - DecisionState, run_signal_loop, ) from litellm.proxy.list_api.common import ( @@ -1286,7 +1285,7 @@ async def _connect_to_count_stored_values() -> SupportsRawQueries: async def _call_current_lens_signal_router( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], @@ -1297,7 +1296,7 @@ async def _call_current_lens_signal_router( decisions: Final[DecisionsCall] = cast(DecisionsCall, current_router.adecisions) return await decisions( model=model, - state=state, + input=input, questions=questions, timeout=timeout, metadata=metadata, diff --git a/tests/unit/proxy/lens/test_signals.py b/tests/unit/proxy/lens/test_signals.py index d2c4eda3966c..9da51eff48b5 100644 --- a/tests/unit/proxy/lens/test_signals.py +++ b/tests/unit/proxy/lens/test_signals.py @@ -19,7 +19,6 @@ SIGNAL_MAX_SCAN_PAGES, SIGNAL_TASK, DecisionQuestions, - DecisionState, Signal, SignalAttempt, SignalClassifier, @@ -45,8 +44,13 @@ LensSampleParams, PartRow, ) -from litellm.types.decisions import DecisionsResponse -from litellm.types.decisions import NoulAnswer as DecisionsNoulAnswer +from litellm.types.decisions import ( + DecisionsInputTokensDetails, + DecisionsOutputTokensDetails, + DecisionsResponse, + DecisionsUsage, +) +from litellm.types.decisions import PredicateAnswer as DecisionsPredicateAnswer NOW: Final = datetime(2026, 10, 7, 12, tzinfo=timezone.utc) CURRENT_CONFIG_KEY: Final = SignalConfig(model="decision").key() @@ -215,6 +219,15 @@ def saved_result(args: tuple[object, ...]) -> SignalData: return SignalData.model_validate_json(payload) +_ZERO_USAGE: Final = DecisionsUsage( + input_tokens=0, + input_tokens_details=DecisionsInputTokensDetails(cached_tokens=0, cache_write_tokens=0), + output_tokens=0, + output_tokens_details=DecisionsOutputTokensDetails(reasoning_tokens=0), + total_tokens=0, +) + + @pytest.mark.asyncio async def test_signal_repository_reads_defaults_and_saves_the_global_config() -> None: database: Final = SignalDatabase(None) @@ -345,34 +358,32 @@ async def test_classifier_sends_noul_questions_and_keeps_every_signal_score() -> async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: assert model == "decision" - assert state == { - "task": SIGNAL_TASK, - "steps": ({"kind": "agent", "name": "agent", "content": "user asks for a result"},), - } - assert _STORED_DATA.validate_json(json.dumps(state)) == { + assert _STORED_DATA.validate_json(input) == { "task": SIGNAL_TASK, "steps": [{"kind": "agent", "name": "agent", "content": "user asks for a result"}], } - expected_questions: Final = { - signal.id: {"type": "noul", "instructions": signal.question} for signal in config.signals - } + expected_questions: Final = tuple( + {"type": "predicate", "name": signal.id, "instructions": signal.question} for signal in config.signals + ) assert questions == expected_questions - assert _STORED_DATA.validate_json(json.dumps(questions)) == expected_questions + assert _STORED_DATA.validate_json(json.dumps(questions)) == list(expected_questions) assert timeout == 60 assert metadata == {"tags": ["litellm-lens-signals"]} return DecisionsResponse( - answers={ - "user_frustration": DecisionsNoulAnswer(type="noul", noul=0.9), - "missing_capability": DecisionsNoulAnswer(type="noul", noul=0.6), - "repeated_request": DecisionsNoulAnswer(type="noul", noul=0.2), - "unknown": DecisionsNoulAnswer(type="noul", noul=1.0), - } + model="decision", + answers=( + DecisionsPredicateAnswer(type="predicate", name="user_frustration", probability=0.9), + DecisionsPredicateAnswer(type="predicate", name="missing_capability", probability=0.6), + DecisionsPredicateAnswer(type="predicate", name="repeated_request", probability=0.2), + DecisionsPredicateAnswer(type="predicate", name="unknown", probability=1.0), + ), + usage=_ZERO_USAGE, ) attempt: Final = await SignalClassifier(SourceReader(storage), decide, lambda: NOW).classify( @@ -387,21 +398,21 @@ async def decide( @pytest.mark.asyncio -async def test_missing_noul_answer_fails_while_unknown_and_non_noul_answers_are_ignored() -> None: +async def test_missing_predicate_answer_fails_while_unknown_and_non_predicate_answers_are_ignored() -> None: async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: return { - "answers": { - "user_frustration": {"type": "noul", "noul": 0.9}, - "missing_capability": {"type": "choice", "choice": "yes"}, - "unknown": {"type": "noul", "noul": 1.0}, - } + "answers": [ + {"type": "predicate", "name": "user_frustration", "probability": 0.9}, + {"type": "choice", "name": "missing_capability", "choice": "yes"}, + {"type": "predicate", "name": "unknown", "probability": 1.0}, + ] } attempt: Final = await SignalClassifier( @@ -412,7 +423,7 @@ async def decide( assert attempt.status == "failed" assert attempt.scores == {"user_frustration": 0.9} - assert attempt.error == "Decisions response omitted a configured noul answer" + assert attempt.error == "Decisions response omitted a configured predicate answer" @pytest.mark.asyncio @@ -420,7 +431,7 @@ async def test_classifier_turns_decisions_errors_into_failed_attempts() -> None: async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], @@ -574,19 +585,19 @@ async def test_signal_tick_classifies_at_most_50_traces_and_persists_scores() -> async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: - steps: Final = TypeAdapter(tuple[SignalStep, ...]).validate_python(state["steps"]) + steps: Final = TypeAdapter(tuple[SignalStep, ...]).validate_python(json.loads(input)["steps"]) await database.calls.put(steps[0].name) return { - "answers": { - "user_frustration": {"type": "noul", "noul": 0.9}, - "missing_capability": {"type": "noul", "noul": 0.6}, - "repeated_request": {"type": "noul", "noul": 0.2}, - } + "answers": [ + {"type": "predicate", "name": "user_frustration", "probability": 0.9}, + {"type": "predicate", "name": "missing_capability", "probability": 0.6}, + {"type": "predicate", "name": "repeated_request", "probability": 0.2}, + ] } await run_signal_tick(storage, repository, decide, lambda: NOW) @@ -648,12 +659,12 @@ def __call__(self) -> datetime: async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: - return {"answers": {}} + return {"answers": []} await run_signal_tick(SignalStorage(executions=executions), repository, decide, AdvancingClock()) @@ -704,12 +715,12 @@ def sample_page(page: int) -> tuple[ExecutionRow, ...]: async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: - return {"answers": {}} + return {"answers": []} first_cursor: Final = await run_signal_tick(storage, repository, decide, lambda: NOW) first_calls: Final = tuple(storage.cursors.get_nowait() for _ in range(storage.cursors.qsize())) @@ -764,17 +775,17 @@ async def test_signal_tick_resumes_a_partially_consumed_page() -> None: async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: return { - "answers": { - "user_frustration": {"type": "noul", "noul": 0.9}, - "missing_capability": {"type": "noul", "noul": 0.6}, - "repeated_request": {"type": "noul", "noul": 0.2}, - } + "answers": [ + {"type": "predicate", "name": "user_frustration", "probability": 0.9}, + {"type": "predicate", "name": "missing_capability", "probability": 0.6}, + {"type": "predicate", "name": "repeated_request", "probability": 0.2}, + ] } first_database: Final = SignalDatabase(config, stored_rows=initial_rows) @@ -835,12 +846,12 @@ async def test_signal_tick_skips_claims_and_writes_when_router_is_not_ready() -> async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: - return {"answers": {}} + return {"answers": []} await run_signal_tick( storage, @@ -865,7 +876,7 @@ async def test_signal_tick_skips_missing_dependencies_and_disabled_configs() -> async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], @@ -902,17 +913,17 @@ async def test_signal_tick_continues_when_storing_a_result_fails() -> None: async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: return { - "answers": { - "user_frustration": {"type": "noul", "noul": 0.9}, - "missing_capability": {"type": "noul", "noul": 0.6}, - "repeated_request": {"type": "noul", "noul": 0.2}, - } + "answers": [ + {"type": "predicate", "name": "user_frustration", "probability": 0.9}, + {"type": "predicate", "name": "missing_capability", "probability": 0.6}, + {"type": "predicate", "name": "repeated_request", "probability": 0.2}, + ] } await run_signal_tick( @@ -943,12 +954,12 @@ async def test_signal_loop_continues_after_a_tick_error() -> None: async def decide( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], ) -> object: - return {"answers": {}} + return {"answers": []} task: Final = asyncio.create_task(run_signal_loop(SignalStorage(), repository, decide, lambda: NOW)) await repository.started.wait() @@ -965,7 +976,7 @@ async def test_proxy_signal_call_resolves_the_current_router(monkeypatch: pytest async def first_decisions( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], @@ -975,7 +986,7 @@ async def first_decisions( async def second_decisions( *, model: str, - state: DecisionState, + input: str, questions: DecisionQuestions, timeout: float, metadata: Mapping[str, object], @@ -985,8 +996,8 @@ async def second_decisions( async def call_current_router() -> object: return await proxy_server._call_current_lens_signal_router( model="decision", - state={"task": "task"}, - questions={}, + input='{"task": "task"}', + questions=(), timeout=60, metadata={"tags": ["test"]}, ) From d3163aa8e40e5c048ea2e376899d4e97857dff77 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:29:54 -0700 Subject: [PATCH 09/15] feat(decisions): refuse safety_identifier for System One providers unless drop_params --- litellm/decisions/main.py | 30 ++++++++++- .../llms/base_llm/decisions/transformation.py | 1 + .../providers/test_decisions_wire.py | 50 +++++++++++++++++++ tests/unit/decisions/test_main.py | 45 +++++++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index b666ab38f124..407b11ba7be4 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -6,6 +6,7 @@ from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig @@ -60,6 +61,28 @@ def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfi return provider_config +def _body_for_provider( + body: DecisionsRequestBody, + provider_config: BaseDecisionsConfig, + *, + model: str, + provider: str, + drop_params: object, +) -> DecisionsRequestBody: + if body.safety_identifier is None or provider_config.supports_safety_identifier: + return body + if litellm.drop_params is True or normalize_drop_params(drop_params) is True: + return body.model_copy(update={"safety_identifier": None}) + raise litellm.UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + model=model, + llm_provider=provider, + ) + + def _prepare_call( *, model: str, @@ -97,7 +120,12 @@ def _prepare_call( model=model, llm_provider=provider, ) from error - request: Final = DecisionsRequest(model=canonical_model, body=body) + request: Final = DecisionsRequest( + model=canonical_model, + body=_body_for_provider( + body, provider_config, model=model, provider=provider, drop_params=kwargs.get("drop_params") + ), + ) resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) if resolved_api_base is None: diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 27a44238b6d0..a6310f23ca06 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -23,6 +23,7 @@ class BaseDecisionsConfig(ABC): api_key_env: tuple[str, ...] = () api_base_env: tuple[str, ...] = () api_key_required: bool = True + supports_safety_identifier: bool = False def get_default_api_base(self) -> str | None: return None diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 6838c81c2431..95e32aac9f78 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -604,3 +604,53 @@ def test_a_deployment_whose_provider_has_no_decisions_support_is_refused_naming_ for provider in _PROVIDERS: assert provider.name in response.text, response.text assert _upstream_calls(gateway, handle) == [] + + +def test_safety_identifier_is_refused_for_system_one_providers_unless_the_deployment_drops_params( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + refused: Final = _decide(gateway, strict, safety_identifier="end-user-7") + assert refused.status_code == 400, refused.text + assert "safety_identifier" in refused.text and "drop_params" in refused.text, refused.text + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = scenario.model( + model=_PERPLEXITY.model, api_base=handle.api_base(), api_key=_PERPLEXITY.api_key, drop_params=True + ) + accepted: Final = _decide(gateway, dropping, safety_identifier="end-user-7") + assert accepted.status_code == 200, accepted.text + assert accepted.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +@pytest.mark.parametrize( + ("questions", "system_one_questions"), + ( + ([], {}), + ( + [{**_PREDICATE, "type": "choice", "choices": []}], + {"q": {"type": "choice", "instructions": "Is it?", "criteria": {}}}, + ), + ( + [{**_PREDICATE, "type": "score", "levels": []}], + {"q": {"type": "score", "instructions": "Is it?", "criteria": []}}, + ), + ), + ids=("no questions", "choice without choices", "score without levels"), +) +def test_empty_collections_reach_the_provider_and_its_verdict_comes_back( + gateway: Gateway, questions: list[JsonValue], system_one_questions: dict[str, JsonValue] +) -> None: + verdict: Final = "at least one criterion is required" + with gateway.scenario() as scenario: + handle: Final = _register(scenario, {"error": {"message": verdict}}, status=400) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, model, questions=questions) + assert response.status_code == 400, response.text + assert verdict in response.text, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {"model": _PERPLEXITY.body_model, "state": _INPUT, "questions": system_one_questions} diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 4b386fafc26b..3ddc02f2c6fc 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -765,3 +765,48 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider") assert route.called assert response.model == "jev-1.13" + + +def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_SYSTEM_ONE_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"safety_identifier.*drop_params") as caught: + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + input=_INPUT, + questions=_predicate(), + safety_identifier="end-user-7", + api_key="caller-key", + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", ("call", "global")) +async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params( + scope: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", scope == "global") + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_SYSTEM_ONE_RESPONSE) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + input=_INPUT, + questions=_predicate(), + safety_identifier="end-user-7", + api_key="caller-key", + **({"drop_params": True} if scope == "call" else {}), + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": _INPUT, + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].model_dump(mode="json") == {"type": "predicate", "name": "is_defect", "probability": 0.9} From d5179127dbaf6bd6441e643bbc9ca3a79a624c1b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 05:59:39 -0700 Subject: [PATCH 10/15] feat(decisions): refuse safety_identifier on providers that cannot take it unless drop_params drops it --- litellm/decisions/main.py | 34 +- .../llms/base_llm/decisions/transformation.py | 1 + .../llms/openai/decisions/transformation.py | 1 + .../test_decisions_openai_format_wire.py | 521 ++++++++++++++++++ .../providers/test_decisions_wire.py | 2 +- .../translation/decisions/bases/cloudflare.py | 78 +++ .../translation/decisions/bases/openrouter.py | 77 +++ .../translation/decisions/bases/perplexity.py | 73 +++ .../decisions/bases/strands_decider.py | 29 + .../translation/decisions/bases/typesafe.py | 73 +++ .../basic/test_decisions_basic_cloudflare.py | 4 +- .../basic/test_decisions_basic_openrouter.py | 9 +- .../basic/test_decisions_basic_perplexity.py | 9 +- .../test_decisions_basic_strands_decider.py | 11 +- .../basic/test_decisions_basic_typesafe.py | 4 +- tests/unit/decisions/test_main.py | 81 +++ 16 files changed, 994 insertions(+), 13 deletions(-) create mode 100644 tests/integration/providers/test_decisions_openai_format_wire.py diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 25ec501fd48d..f75d7a322b7d 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Final, TypeAlias import httpx @@ -7,6 +7,7 @@ from typing_extensions import assert_never import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( @@ -105,6 +106,33 @@ def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest: assert_never(request) +def _drops_params(kwargs: Mapping[str, object]) -> bool: + return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True + + +def _provider_ir_request( + request: DecisionsRequestFormat, + *, + model: str, + provider: str, + provider_config: BaseDecisionsConfig, + kwargs: Mapping[str, object], +) -> DecisionsIRRequest: + ir_request: Final = _ir_request(request) + if ir_request.safety_identifier is None or provider_config.supports_safety_identifier: + return ir_request + if _drops_params(kwargs): + return replace(ir_request, safety_identifier=None) + raise litellm.UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + model=model, + llm_provider=provider, + ) + + def _prepare_call( *, model: str, @@ -165,7 +193,9 @@ def _prepare_call( llm_provider=provider, ) - ir_request: Final = _ir_request(request) + ir_request: Final = _provider_ir_request( + request, model=model, provider=provider, provider_config=provider_config, kwargs=kwargs + ) body: Final = provider_config.transform_decisions_request( model=canonical_model, request=ir_request, custom_llm_provider=provider ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index a8d14ca59a3d..e9a5269c06ee 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -301,6 +301,7 @@ class BaseDecisionsConfig(ABC): api_key_env: tuple[str, ...] = () api_base_env: tuple[str, ...] = () api_key_required: bool = True + supports_safety_identifier: bool = False def get_default_api_base(self) -> str | None: return None diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 0afbe1a434e6..317e5abeeb5b 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -292,6 +292,7 @@ def ir_to_openai_response( class OpenAIDecisionsConfig(BaseDecisionsConfig): path = "/v1/decisions" + supports_safety_identifier = True def get_default_api_base(self) -> str | None: return "https://api.openai.com" diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py new file mode 100644 index 000000000000..b92928c592cd --- /dev/null +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -0,0 +1,521 @@ +import math +import uuid +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse +from pydantic import JsonValue, TypeAdapter + +import litellm + +_API_KEY: Final = "synthetic-decisions-key" +_SAFETY_IDENTIFIER: Final = "end-user-7" +_DROPPING_DEPLOYMENT: Final = "decisions-under-litellm-settings-drop-params" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_INPUT: Final = "Ticket (billing): The export job hangs at 99%" +_FOLLOW_UP: Final = "Customer: still stuck after retrying" +_QUESTIONS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [{"value": "low", "description": "cosmetic"}, {"value": "high", "description": "blocks users"}], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, +] +_SYSTEM_ONE_QUESTIONS: Final[dict[str, JsonValue]] = { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, +} +_SYSTEM_ONE_ANSWERS: Final[dict[str, JsonValue]] = { + "defect": {"type": "noul", "noul": 0.93}, + "severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}}, + "confidence": { + "type": "score", + "score": 1.0, + "confidence": 0.7, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.3, "1": 0.7}, + }, +} +_ANSWERS: Final[list[JsonValue]] = [ + {"type": "predicate", "name": "defect", "probability": 0.93}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [{"value": "low", "probability": 0.2}, {"value": "high", "probability": 0.8}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "confidence", + "score": 1.0, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.3}, + {"value": 1, "label": "sure", "probability": 0.7}, + ], + "confidence": 0.7, + }, +] +_INPUT_TOKENS: Final = 367 +_OUTPUT_TOKENS: Final = 3 +_SYSTEM_ONE_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS} +_USAGE: Final[dict[str, JsonValue]] = { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, +} +_SDK_QUESTIONS: Final = TypeAdapter(list[dict[str, object]]).validate_python(_QUESTIONS) +_SPEND_QUERY: Final = ( + "SELECT spend, status, call_type, model_group, custom_llm_provider, api_base, prompt_tokens, completion_tokens, " + 'request_tags FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) + + +@dataclass(frozen=True, slots=True) +class _Provider: + name: str + model: str + path: str + body_model: str + api_key: str | None + wraps_result: bool + cost_map_key: str | None + speaks_openai: bool = False + + def upstream_body(self) -> dict[str, JsonValue]: + if self.speaks_openai: + return {"model": self.body_model, "input": _INPUT, "questions": _QUESTIONS} + return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + + def upstream_reply(self) -> dict[str, JsonValue]: + if self.speaks_openai: + return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + answer: Final[dict[str, JsonValue]] = { + "model": self.body_model, + "answers": _SYSTEM_ONE_ANSWERS, + "usage": _SYSTEM_ONE_USAGE, + } + return {"result": answer, "success": True} if self.wraps_result else answer + + def litellm_response(self) -> dict[str, JsonValue]: + return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + + +_PROVIDERS: Final = ( + _Provider( + "perplexity", + "perplexity/pplx-decider-v1-27b", + "/v1/decisions", + "pplx-decider-v1-27b", + _API_KEY, + False, + "perplexity/pplx-decider-v1-27b", + ), + _Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"), + _Provider( + "openrouter", + "openrouter/typesafe/jev-1.13", + "/alpha/decisions", + "typesafe/jev-1.13", + _API_KEY, + False, + "openrouter/typesafe/jev-1.13", + ), + _Provider( + "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None + ), + _Provider( + "cloudflare", + "cloudflare/clef", + "/ai/run/@cf/cloudflare/clef", + "clef", + _API_KEY, + True, + "cloudflare/@cf/cloudflare/clef", + ), + _Provider("openai", "openai/gpt-6-luna", "/v1/decisions", "gpt-6-luna", _API_KEY, False, "gpt-6-luna", True), +) +_SYSTEM_ONE_PROVIDERS: Final = _PROVIDERS[:5] +_PERPLEXITY: Final = _PROVIDERS[0] +_TYPESAFE: Final = _PROVIDERS[1] +_OPENAI: Final = _PROVIDERS[5] +_PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} +_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( + ("missing questions", {"input": _INPUT}), + ("missing input", {"questions": _QUESTIONS}), + ("numeric input", {"input": 5, "questions": _QUESTIONS}), + ("assistant message input", {"input": [{"role": "assistant", "content": "hi"}], "questions": _QUESTIONS}), + ("empty questions", {"input": _INPUT, "questions": []}), + ("questions as a map", {"input": _INPUT, "questions": {"q": _PREDICATE}}), + ("predicate without instructions", {"input": _INPUT, "questions": [{"type": "predicate", "name": "q"}]}), + ( + "choice with one choice", + {"input": _INPUT, "questions": [{**_PREDICATE, "type": "choice", "choices": [{"value": "only"}]}]}, + ), + ( + "score with one level", + {"input": _INPUT, "questions": [{**_PREDICATE, "type": "score", "levels": [{"label": "only"}]}]}, + ), + ("unknown question type", {"input": _INPUT, "questions": [{**_PREDICATE, "type": "ranking"}]}), + ("numeric safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": 7}), + ("list safety_identifier", {"input": _INPUT, "questions": _QUESTIONS, "safety_identifier": [_SAFETY_IDENTIFIER]}), +) +_IMAGE_INPUT: Final[list[JsonValue]] = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": _INPUT}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } +] +_OPENAI_IMAGE_MESSAGES: Final[list[JsonValue]] = [ + { + "role": "user", + "type": "message", + "content": [ + {"type": "input_text", "text": _INPUT}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } +] +_MESSAGE_LIST_INPUT: Final[list[JsonValue]] = [ + {"role": "user", "content": _INPUT}, + {"role": "user", "content": [{"type": "input_text", "text": _FOLLOW_UP}]}, +] + + +def _response_cost(response: httpx.Response) -> float: + return float(response.headers["x-litellm-response-cost"]) if "x-litellm-response-cost" in response.headers else 0.0 + + +def _provider_id(provider: _Provider) -> str: + return provider.name + + +def _number(value: JsonValue) -> float: + assert isinstance(value, (int, float)) and not isinstance(value, bool), value + return float(value) + + +def _expected_spend(cost_map_key: str | None) -> float: + if cost_map_key is None: + return 0.0 + cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) + prices: Final = object_value(cost_map[cost_map_key]) + return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number( + prices["output_cost_per_token"] + ) + + +def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle: + handle: Final = register_scenario( + f"decisions-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status) + ) + scenario.cleanups.callback(delete_scenario, handle) + return handle + + +def _deployment(scenario: Scenario, handle: ScenarioHandle, provider: _Provider, *, drop_params: bool = False) -> str: + if drop_params: + return scenario.model( + model=provider.model, api_base=handle.api_base(), api_key=provider.api_key, drop_params=True + ) + return scenario.model(model=provider.model, api_base=handle.api_base(), api_key=provider.api_key) + + +def _decide(gateway: Gateway, model: str, *, route: str = "/v1/decisions", **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", route, {"model": model, "input": _INPUT, "questions": _QUESTIONS, **extra}) + + +def _observed_requests(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]: + with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: + observations: Final = _JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"] + assert isinstance(observations, list), observations + return tuple(map(object_value, observations)) + + +def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> list[dict[str, JsonValue]]: + return [request for request in requests if string_value(request["path"]).startswith(f"/{handle.scenario_id}/")] + + +def _upstream_calls(gateway: Gateway, handle: ScenarioHandle) -> list[dict[str, JsonValue]]: + return _calls_to(_observed_requests(gateway), handle) + + +def _spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70) + return rows[0] + + +def _assert_refused_as_invalid(gateway: Gateway, model: str, label: str, body: dict[str, JsonValue]) -> None: + response: Final = gateway.request("POST", "/v1/decisions", {"model": model, **body}) + assert response.status_code == 400, (label, response.text) + assert "Invalid Decisions request" in response.text, (label, response.text) + + +def _assert_refused_for_safety_identifier(response: httpx.Response) -> None: + assert response.status_code == 400, response.text + assert "safety_identifier" in response.text and "drop_params" in response.text, response.text + + +def _drop_params_config(directory: Path, api_base: str) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **base, + "litellm_settings": {**object_value(base["litellm_settings"]), "drop_params": True}, + "model_list": [ + { + "model_name": _DROPPING_DEPLOYMENT, + "litellm_params": {"model": _PERPLEXITY.model, "api_base": api_base, "api_key": _API_KEY}, + } + ], + } + path: Final = directory / "decisions-drop-params.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.mark.parametrize("provider", _PROVIDERS, ids=_provider_id) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map( + gateway: Gateway, provider: _Provider +) -> None: + expected_spend: Final = _expected_spend(provider.cost_map_key) + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + response: Final = _decide(gateway, model) + assert response.status_code == 200, response.text + assert response.json() == provider.litellm_response() + assert response.headers["x-litellm-model-group"] == model + assert math.isclose(_response_cost(response), expected_spend, rel_tol=1e-9) + (call,) = _upstream_calls(gateway, handle) + assert call["path"] == f"/{handle.scenario_id}{provider.path}" + assert call["authorization"] == (f"Bearer {provider.api_key}" if provider.api_key else "") + assert call["body"] == provider.upstream_body() + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert ( + row["status"], + row["call_type"], + row["custom_llm_provider"], + row["model_group"], + row["api_base"], + row["prompt_tokens"], + row["completion_tokens"], + ) == ( + "success", + "adecisions", + provider.name, + model, + f"{handle.api_base()}{provider.path}", + _INPUT_TOKENS, + _OUTPUT_TOKENS, + ) + assert math.isclose(_number(row["spend"]), expected_spend, rel_tol=1e-9), row + + +def test_the_unversioned_alias_serves_the_same_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, model, route="/decisions") + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_repeated_identical_requests_each_reach_the_upstream_and_are_each_billed(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + responses: Final = tuple(_decide(gateway, model) for _ in range(2)) + assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses] + call_ids: Final = tuple(response.headers["x-litellm-call-id"] for response in responses) + assert len(set(call_ids)) == 2, call_ids + assert len(_upstream_calls(gateway, handle)) == 2 + for call_id in call_ids: + assert _spend_row(call_id)["status"] == "success" + + +async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _TYPESAFE.upstream_reply()) + synchronous: Final = litellm.decisions( + model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + ) + asynchronous: Final = await litellm.adecisions( + model=_TYPESAFE.model, input=_INPUT, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + ) + for response in (synchronous, asynchronous): + assert response.model_dump(mode="json") == _TYPESAFE.litellm_response() + calls: Final = _upstream_calls(gateway, handle) + assert len(calls) == 2, calls + for call in calls: + assert call["path"] == f"/{handle.scenario_id}{_TYPESAFE.path}" + assert call["authorization"] == f"Bearer {_API_KEY}" + assert call["body"] == _TYPESAFE.upstream_body() + + +def test_gateway_only_fields_stay_at_the_gateway_and_tags_reach_the_spend_log(gateway: Gateway) -> None: + tag: Final = f"decisions-openai-format-{uuid.uuid4().hex[:8]}" + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide( + gateway, model, user="auditor", num_retries=0, temperature=0.2, metadata={"tags": [tag]} + ) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + tags: Final = row["request_tags"] + assert isinstance(tags, list) and tag in tags, row + + +def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + for label, body in _INVALID_BODIES: + _assert_refused_as_invalid(gateway, model, label, body) + assert _upstream_calls(gateway, handle) == [] + + +@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id) +def test_system_one_providers_refuse_images_at_the_gateway_without_an_upstream_call( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + model: Final = _deployment(scenario, handle, provider) + response: Final = _decide(gateway, model, input=_IMAGE_INPUT) + assert response.status_code == 400, response.text + assert "input_image" in response.text + assert _upstream_calls(gateway, handle) == [] + + +def test_openai_forwards_image_input_in_its_own_message_shape(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide(gateway, model, input=_IMAGE_INPUT) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {"model": _OPENAI.body_model, "input": _OPENAI_IMAGE_MESSAGES, "questions": _QUESTIONS} + + +def test_a_message_list_input_reaches_a_system_one_provider_as_its_flattened_text(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + model: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, model, input=_MESSAGE_LIST_INPUT) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == { + "model": _PERPLEXITY.body_model, + "state": f"{_INPUT}\n\n{_FOLLOW_UP}", + "questions": _SYSTEM_ONE_QUESTIONS, + } + row: Final = _spend_row(response.headers["x-litellm-call-id"]) + assert (row["status"], row["call_type"], row["prompt_tokens"], row["completion_tokens"]) == ( + "success", + "adecisions", + _INPUT_TOKENS, + _OUTPUT_TOKENS, + ) + + +@pytest.mark.parametrize("provider", _SYSTEM_ONE_PROVIDERS, ids=_provider_id) +def test_safety_identifier_is_refused_by_system_one_providers_unless_the_deployment_drops_params( + gateway: Gateway, provider: _Provider +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, provider.upstream_reply()) + strict: Final = _deployment(scenario, handle, provider) + refused: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER) + _assert_refused_for_safety_identifier(refused) + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, provider, drop_params=True) + accepted: Final = _decide(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER) + assert accepted.status_code == 200, accepted.text + assert accepted.json() == provider.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == provider.upstream_body() + row: Final = _spend_row(accepted.headers["x-litellm-call-id"]) + assert (row["status"], row["call_type"], row["model_group"]) == ("success", "adecisions", dropping) + + +@pytest.mark.parametrize("value", ("", "x" * 5000), ids=("empty", "5kb")) +def test_every_string_safety_identifier_is_refused_or_dropped_like_the_usual_one(gateway: Gateway, value: str) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + _assert_refused_for_safety_identifier(_decide(gateway, strict, safety_identifier=value)) + dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True) + accepted: Final = _decide(gateway, dropping, safety_identifier=value) + assert accepted.status_code == 200, accepted.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_a_request_body_drop_params_drops_the_safety_identifier_like_chat(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + response: Final = _decide(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER, drop_params=True) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_openai_keeps_the_safety_identifier_on_the_wire_without_drop_params(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide(gateway, model, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + assert response.json() == _OPENAI.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} + + +def test_litellm_settings_drop_params_drops_the_safety_identifier_for_a_strict_deployment( + gateway: Gateway, tmp_path: Path +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + config: Final = _drop_params_config(tmp_path, handle.api_base()) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + response: Final = _decide(owned.gateway, _DROPPING_DEPLOYMENT, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + assert response.json() == _PERPLEXITY.litellm_response() + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 84b31088d608..547cefd7b2e0 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -107,7 +107,7 @@ class _Provider: _PERPLEXITY: Final = _PROVIDERS[0] _OPENROUTER: Final = _PROVIDERS[2] _OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini" -_UNSUPPORTED_PROVIDER_MODEL: Final = "openai/gpt-6-luna" +_UNSUPPORTED_PROVIDER_MODEL: Final = "anthropic/claude-opus-4-1" _CONNECTION_ERROR: Final = "litellm.APIConnectionError" _GENERIC_API_ERROR: Final = "litellm.APIError" _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py index e726c382abf7..3efc96eeb878 100644 --- a/tests/integration/translation/decisions/bases/cloudflare.py +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -101,3 +101,81 @@ }, }, ) +CLEF_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "cloudflare/@cf/cloudflare/clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", + expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "result": { + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + expected_litellm_response={ + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, +) diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py index 8c42a7ded503..ed39bc1a55be 100644 --- a/tests/integration/translation/decisions/bases/openrouter.py +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -98,3 +98,80 @@ }, }, ) +TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "openrouter/typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/alpha/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, + expected_litellm_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, +) diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py index f9b4d029467e..9e29ceab8ccd 100644 --- a/tests/integration/translation/decisions/bases/perplexity.py +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -99,3 +99,76 @@ }, }, ) +PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "perplexity/pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, +) diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py index 7983d4e65913..2950b0907377 100644 --- a/tests/integration/translation/decisions/bases/strands_decider.py +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -38,3 +38,32 @@ }, }, ) +STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "strands_decider/strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + }, + mock_provider_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, + expected_litellm_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, +) diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py index 2e877ca83a1d..7d47ce6b0827 100644 --- a/tests/integration/translation/decisions/bases/typesafe.py +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -96,3 +96,76 @@ }, }, ) +JEV_1_13_0_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "typesafe/jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, + expected_litellm_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, +) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py index 79fc0574b3bb..21e0538897d5 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py @@ -2,10 +2,10 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE +from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [CLEF_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [CLEF_TEST_CASE, CLEF_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py index a74cb23f6b43..973526d72729 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py @@ -2,10 +2,15 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.openrouter import TYPESAFE_JEV_1_13_TEST_CASE +from integration.translation.decisions.bases.openrouter import ( + TYPESAFE_JEV_1_13_TEST_CASE, + TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE, +) from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [TYPESAFE_JEV_1_13_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize( + "case", [TYPESAFE_JEV_1_13_TEST_CASE, TYPESAFE_JEV_1_13_SYSTEMONE_TEST_CASE], ids=lambda case: case.id +) def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py index e341d24ba537..d9662ec4b9f8 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py @@ -2,10 +2,15 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.perplexity import PPLX_DECIDER_V1_27B_TEST_CASE +from integration.translation.decisions.bases.perplexity import ( + PPLX_DECIDER_V1_27B_TEST_CASE, + PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE, +) from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [PPLX_DECIDER_V1_27B_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize( + "case", [PPLX_DECIDER_V1_27B_TEST_CASE, PPLX_DECIDER_V1_27B_SYSTEMONE_TEST_CASE], ids=lambda case: case.id +) def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py index cae7ca70c780..2d2cd9caaf54 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py @@ -2,10 +2,17 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.strands_decider import STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE +from integration.translation.decisions.bases.strands_decider import ( + STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, + STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE, +) from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize( + "case", + [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE, STRANDS_DECIDER_2B_HOBSON_V19_SYSTEMONE_TEST_CASE], + ids=lambda case: case.id, +) def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py index 302636e6fb41..c1111cc6ba95 100644 --- a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py @@ -2,10 +2,10 @@ from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import TranslationTestCase -from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE +from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE from integration.translation.runner import assert_translation -@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE], ids=lambda case: case.id) +@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE, JEV_1_13_0_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 820451ed9431..72cb40911e23 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -922,3 +922,84 @@ async def test_requests_a_provider_cannot_serve_are_rejected_before_http( await litellm.adecisions(model=model, api_key="caller-key", **request_kwargs) assert len(respx_mock.calls) == 0 + + +_SAFETY_IDENTIFIER: Final = "end-user-7" +_PREDICATE_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, +) + + +def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_it( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + with pytest.raises(litellm.UnsupportedParamsError, match=r"safety_identifier.*drop_params") as caught: + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + input="review", + questions=_PREDICATE_QUESTIONS, + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + ) + + assert caught.value.status_code == 400 + assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", ("call", "global")) +async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params( + scope: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", scope == "global") + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + input="review", + questions=_PREDICATE_QUESTIONS, + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + **({"drop_params": True} if scope == "call" else {}), + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "is_defect", "probability": 0.9} + ] + + +@pytest.mark.asyncio +async def test_safety_identifier_reaches_openai_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + + await litellm.adecisions( + model="openai/gpt-6-luna", + input="review", + questions=_PREDICATE_QUESTIONS, + safety_identifier=_SAFETY_IDENTIFIER, + api_key="caller-key", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + "safety_identifier": _SAFETY_IDENTIFIER, + } From e547414e1df007e24b3a3f3072376d34c63256f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:58:09 -0700 Subject: [PATCH 11/15] test(decisions): cover hosted_vllm in the safety_identifier and per-provider wire tests --- .../providers/test_decisions_openai_format_wire.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index b92928c592cd..367c3690e31c 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -157,12 +157,13 @@ def litellm_response(self) -> dict[str, JsonValue]: True, "cloudflare/@cf/cloudflare/clef", ), + _Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None), _Provider("openai", "openai/gpt-6-luna", "/v1/decisions", "gpt-6-luna", _API_KEY, False, "gpt-6-luna", True), ) -_SYSTEM_ONE_PROVIDERS: Final = _PROVIDERS[:5] +_SYSTEM_ONE_PROVIDERS: Final = tuple(provider for provider in _PROVIDERS if not provider.speaks_openai) _PERPLEXITY: Final = _PROVIDERS[0] _TYPESAFE: Final = _PROVIDERS[1] -_OPENAI: Final = _PROVIDERS[5] +_OPENAI: Final = next(provider for provider in _PROVIDERS if provider.speaks_openai) _PREDICATE: Final[dict[str, JsonValue]] = {"type": "predicate", "name": "q", "instructions": "Is it?"} _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("missing questions", {"input": _INPUT}), From 6ed3e6591001a5ec7ce6e9e4e2953508736781b8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:30:34 -0700 Subject: [PATCH 12/15] fix(decisions): check a System One request's safety_identifier against the provider too --- litellm/decisions/main.py | 20 ++++++--- .../test_decisions_openai_format_wire.py | 45 +++++++++++++++++++ tests/unit/decisions/test_main.py | 33 +++++++++++--- 3 files changed, 87 insertions(+), 11 deletions(-) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index c60670a7a6de..12b4c5beac8e 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -3,7 +3,7 @@ from typing import Final, TypeAlias import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import assert_never import litellm @@ -40,6 +40,9 @@ _SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( + str | None, config=ConfigDict(title="safety_identifier") +) _HANDLER: Final = BaseLLMHTTPHandler() @@ -96,10 +99,10 @@ def _validate_request( ) -def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest: +def _ir_request(request: DecisionsRequestFormat, safety_identifier: str | None) -> DecisionsIRRequest: match request: case DecisionsRequestBody(): - return systemone_request_to_ir(request) + return replace(systemone_request_to_ir(request), safety_identifier=safety_identifier) case OpenAIDecisionRequestBody(): return openai_request_to_ir(request) case _: @@ -113,12 +116,13 @@ def _drops_params(kwargs: Mapping[str, object]) -> bool: def _provider_ir_request( request: DecisionsRequestFormat, *, + safety_identifier: str | None, model: str, provider: str, provider_config: BaseDecisionsConfig, kwargs: Mapping[str, object], ) -> DecisionsIRRequest: - ir_request: Final = _ir_request(request) + ir_request: Final = _ir_request(request, safety_identifier) if ir_request.safety_identifier is None or provider_config.supports_safety_identifier: return ir_request if _drops_params(kwargs): @@ -171,6 +175,7 @@ def _prepare_call( request: Final = _validate_request( state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier ) + validated_safety_identifier: Final = _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) except ValidationError as error: raise litellm.BadRequestError( message=f"Invalid Decisions request: {error}", @@ -194,7 +199,12 @@ def _prepare_call( ) ir_request: Final = _provider_ir_request( - request, model=model, provider=provider, provider_config=provider_config, kwargs=kwargs + request, + safety_identifier=validated_safety_identifier, + model=model, + provider=provider, + provider_config=provider_config, + kwargs=kwargs, ) body: Final = provider_config.transform_decisions_request( model=canonical_model, request=ir_request, custom_llm_provider=provider diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index 367c3690e31c..15bbe0bcc2ae 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -253,6 +253,12 @@ def _decide(gateway: Gateway, model: str, *, route: str = "/v1/decisions", **ext return gateway.request("POST", route, {"model": model, "input": _INPUT, "questions": _QUESTIONS, **extra}) +def _decide_in_system_one_format(gateway: Gateway, model: str, **extra: JsonValue) -> httpx.Response: + return gateway.request( + "POST", "/v1/systemone", {"model": model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS, **extra} + ) + + def _observed_requests(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]: with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: observations: Final = _JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"] @@ -508,6 +514,45 @@ def test_openai_keeps_the_safety_identifier_on_the_wire_without_drop_params(gate assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} +def test_a_system_one_format_safety_identifier_is_refused_unless_the_deployment_drops_params(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _PERPLEXITY.upstream_reply()) + strict: Final = _deployment(scenario, handle, _PERPLEXITY) + refused: Final = _decide_in_system_one_format(gateway, strict, safety_identifier=_SAFETY_IDENTIFIER) + _assert_refused_for_safety_identifier(refused) + assert _upstream_calls(gateway, handle) == [] + + dropping: Final = _deployment(scenario, handle, _PERPLEXITY, drop_params=True) + accepted: Final = _decide_in_system_one_format(gateway, dropping, safety_identifier=_SAFETY_IDENTIFIER) + assert accepted.status_code == 200, accepted.text + assert accepted.json()["answers"] == _SYSTEM_ONE_ANSWERS + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _PERPLEXITY.upstream_body() + + +def test_openai_keeps_a_system_one_format_safety_identifier_on_the_wire(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide_in_system_one_format(gateway, model, safety_identifier=_SAFETY_IDENTIFIER) + assert response.status_code == 200, response.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} + + +@pytest.mark.parametrize("value", (7, [_SAFETY_IDENTIFIER]), ids=("numeric", "list")) +def test_a_non_string_system_one_format_safety_identifier_is_refused_as_invalid( + gateway: Gateway, value: JsonValue +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _OPENAI.upstream_reply()) + model: Final = _deployment(scenario, handle, _OPENAI) + response: Final = _decide_in_system_one_format(gateway, model, safety_identifier=value) + assert response.status_code == 400, response.text + assert "Invalid Decisions request" in response.text, response.text + assert _upstream_calls(gateway, handle) == [] + + def test_litellm_settings_drop_params_drops_the_safety_identifier_for_a_strict_deployment( gateway: Gateway, tmp_path: Path ) -> None: diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 853de3424a1f..de4448c4f71c 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1092,10 +1092,16 @@ async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map _PREDICATE_QUESTIONS: Final[tuple[Mapping[str, object], ...]] = ( {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, ) +_PREDICATE_REQUESTS: Final[tuple[Mapping[str, object], ...]] = ( + MappingProxyType({"input": "review", "questions": _PREDICATE_QUESTIONS}), + MappingProxyType({"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}}), +) +_PREDICATE_REQUEST_IDS: Final = ("input_format", "state_format") +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_it( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) @@ -1103,10 +1109,9 @@ def test_safety_identifier_is_refused_before_http_when_the_provider_cannot_take_ with pytest.raises(litellm.UnsupportedParamsError, match=r"safety_identifier.*drop_params") as caught: litellm.decisions( model="perplexity/pplx-decider-v1-27b", - input="review", - questions=_PREDICATE_QUESTIONS, safety_identifier=_SAFETY_IDENTIFIER, api_key="caller-key", + **request_kwargs, ) assert caught.value.status_code == 400 @@ -1143,8 +1148,9 @@ async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params( @pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) async def test_safety_identifier_reaches_openai_without_drop_params( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "drop_params", False) monkeypatch.setattr(litellm, "api_base", None) @@ -1154,10 +1160,9 @@ async def test_safety_identifier_reaches_openai_without_drop_params( await litellm.adecisions( model="openai/gpt-6-luna", - input="review", - questions=_PREDICATE_QUESTIONS, safety_identifier=_SAFETY_IDENTIFIER, api_key="caller-key", + **request_kwargs, ) assert route.called @@ -1167,3 +1172,19 @@ async def test_safety_identifier_reaches_openai_without_drop_params( "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], "safety_identifier": _SAFETY_IDENTIFIER, } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +async def test_non_string_safety_identifier_is_rejected_before_http( + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + invalid_safety_identifier: Final[Mapping[str, object]] = MappingProxyType({"safety_identifier": 7}) + + with pytest.raises(litellm.BadRequestError, match=r"(?s)Invalid Decisions request.*safety_identifier"): + await litellm.adecisions( + model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **invalid_safety_identifier + ) + + assert not route.called From 48fd7420eaf498764c1033c14950c9d5dac4914e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:07:00 -0700 Subject: [PATCH 13/15] fix(decisions): drop a non-string safety_identifier under drop_params A malformed safety_identifier now follows the drop_params convention in both body shapes: it answers 400 without drop_params and is dropped before the provider call with it. A string identifier on OpenAI stays on the wire either way. --- litellm/decisions/main.py | 15 +++++-- .../test_decisions_openai_format_wire.py | 21 ++++++--- tests/unit/decisions/test_main.py | 43 ++++++++++++++++--- 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 12b4c5beac8e..76ef8e272961 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -113,6 +113,12 @@ def _drops_params(kwargs: Mapping[str, object]) -> bool: return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True +def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None: + if isinstance(safety_identifier, str) or not _drops_params(kwargs): + return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) + return None + + def _provider_ir_request( request: DecisionsRequestFormat, *, @@ -172,10 +178,13 @@ def _prepare_call( llm_provider=provider, ) try: + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs) request: Final = _validate_request( - state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier + state=state, + questions=questions, + decision_input=decision_input, + safety_identifier=request_safety_identifier, ) - validated_safety_identifier: Final = _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) except ValidationError as error: raise litellm.BadRequestError( message=f"Invalid Decisions request: {error}", @@ -200,7 +209,7 @@ def _prepare_call( ir_request: Final = _provider_ir_request( request, - safety_identifier=validated_safety_identifier, + safety_identifier=request_safety_identifier, model=model, provider=provider, provider_config=provider_config, diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index 15bbe0bcc2ae..357bfbbfe59b 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -1,6 +1,6 @@ import math import uuid -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from pathlib import Path from typing import Final @@ -540,18 +540,25 @@ def test_openai_keeps_a_system_one_format_safety_identifier_on_the_wire(gateway: assert call["body"] == {**_OPENAI.upstream_body(), "safety_identifier": _SAFETY_IDENTIFIER} +@pytest.mark.parametrize("decide", (_decide, _decide_in_system_one_format), ids=("openai_format", "system_one_format")) @pytest.mark.parametrize("value", (7, [_SAFETY_IDENTIFIER]), ids=("numeric", "list")) -def test_a_non_string_system_one_format_safety_identifier_is_refused_as_invalid( - gateway: Gateway, value: JsonValue +def test_a_non_string_safety_identifier_is_refused_as_invalid_unless_the_deployment_drops_params( + gateway: Gateway, value: JsonValue, decide: Callable[..., httpx.Response] ) -> None: with gateway.scenario() as scenario: handle: Final = _register(scenario, _OPENAI.upstream_reply()) - model: Final = _deployment(scenario, handle, _OPENAI) - response: Final = _decide_in_system_one_format(gateway, model, safety_identifier=value) - assert response.status_code == 400, response.text - assert "Invalid Decisions request" in response.text, response.text + strict: Final = _deployment(scenario, handle, _OPENAI) + refused: Final = decide(gateway, strict, safety_identifier=value) + assert refused.status_code == 400, refused.text + assert "Invalid Decisions request" in refused.text, refused.text assert _upstream_calls(gateway, handle) == [] + dropping: Final = _deployment(scenario, handle, _OPENAI, drop_params=True) + accepted: Final = decide(gateway, dropping, safety_identifier=value) + assert accepted.status_code == 200, accepted.text + (call,) = _upstream_calls(gateway, handle) + assert call["body"] == _OPENAI.upstream_body() + def test_litellm_settings_drop_params_drops_the_safety_identifier_for_a_strict_deployment( gateway: Gateway, tmp_path: Path diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index de4448c4f71c..eaec67dd974d 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1094,7 +1094,9 @@ async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map ) _PREDICATE_REQUESTS: Final[tuple[Mapping[str, object], ...]] = ( MappingProxyType({"input": "review", "questions": _PREDICATE_QUESTIONS}), - MappingProxyType({"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}}), + MappingProxyType( + {"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}} + ), ) _PREDICATE_REQUEST_IDS: Final = ("input_format", "state_format") @@ -1149,10 +1151,14 @@ async def test_safety_identifier_is_dropped_from_the_wire_under_drop_params( @pytest.mark.asyncio @pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) -async def test_safety_identifier_reaches_openai_without_drop_params( - request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("drop_params", (False, True), ids=("strict", "drop_params")) +async def test_safety_identifier_reaches_openai_whether_or_not_params_are_dropped( + drop_params: bool, + request_kwargs: Mapping[str, object], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "drop_params", drop_params) monkeypatch.setattr(litellm, "api_base", None) monkeypatch.delenv("OPENAI_BASE_URL", raising=False) monkeypatch.delenv("OPENAI_API_BASE", raising=False) @@ -1177,8 +1183,9 @@ async def test_safety_identifier_reaches_openai_without_drop_params( @pytest.mark.asyncio @pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) async def test_non_string_safety_identifier_is_rejected_before_http( - request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) invalid_safety_identifier: Final[Mapping[str, object]] = MappingProxyType({"safety_identifier": 7}) @@ -1188,3 +1195,29 @@ async def test_non_string_safety_identifier_is_rejected_before_http( ) assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) +async def test_non_string_safety_identifier_is_dropped_from_the_wire_under_drop_params( + request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + dropped_safety_identifier: Final[Mapping[str, object]] = MappingProxyType( + {"safety_identifier": 7, "drop_params": True} + ) + + await litellm.adecisions( + model="openai/gpt-6-luna", api_key="caller-key", **request_kwargs, **dropped_safety_identifier + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } From 21d083399a94797d0b93bf389d7361c2b86896d7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:40:32 -0700 Subject: [PATCH 14/15] fix(decisions): let /v1/decisions drop a non-string safety_identifier under drop_params The route checked the whole body before routing, so a non-string safety_identifier answered 400 even when the deployment or the body set drop_params. The route now leaves that field to the Decisions call, which knows every drop_params source --- .../proxy/decisions_endpoints/endpoints.py | 7 ++- .../decisions_endpoints/test_endpoints.py | 62 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index f2891c184bf6..4f4209208aca 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -1,4 +1,5 @@ from collections.abc import Mapping +from types import MappingProxyType from typing import Annotated, Final from fastapi import APIRouter, Depends, Request, Response @@ -38,6 +39,10 @@ async def _invalid_request( ) +def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in data.items() if key != "safety_identifier"}) + + async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: body: Final = await request.body() try: @@ -75,7 +80,7 @@ async def _process_decisions( data: Final = await _request_data(request, user_api_key_dict) try: - body_adapter.validate_python(data) + body_adapter.validate_python(_fields_checked_before_routing(data)) except ValidationError as error: raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index ccfa12e10d62..e2f7b6ab8844 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -530,6 +530,68 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) +@pytest.mark.parametrize("safety_identifier", (7, ["end-user-1"]), ids=("numeric", "list")) +@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) +def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + safety_identifier: object, + deployment_drops_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "decider", + "litellm_params": { + "model": "openai/gpt-6-luna", + "api_key": "k", + "drop_params": deployment_drops_params, + }, + } + ] + ), + ) + upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={ + "model": "gpt-6-luna", + "answers": _OPENAI_FORMAT_ANSWERS[:1], + "usage": { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, + } + ) + request_body: Final = { + "model": "decider", + "input": "The package arrived with a broken screen.", + "questions": _OPENAI_FORMAT_REQUEST["questions"][:1], + "safety_identifier": safety_identifier, + } + + response: Final = client.post("/v1/decisions", json=request_body) + + if not deployment_drops_params: + assert response.status_code == 400, response.text + assert "safety_identifier" in response.json()["error"]["message"] + assert not upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived with a broken screen.", + "questions": _OPENAI_FORMAT_REQUEST["questions"][:1], + } + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions") From 10080051a5fa944b364a69ea3d54dd7cbf0ad5b5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:45:19 -0700 Subject: [PATCH 15/15] refactor(decisions): return the unsupported safety_identifier as a value and raise it in _prepare_call --- litellm/decisions/main.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 22543babb3e3..032cad004544 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -119,28 +119,24 @@ def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, o return None +@dataclass(frozen=True, slots=True) +class _UnsupportedSafetyIdentifier: + pass + + def _provider_ir_request( request: DecisionsRequestFormat, *, safety_identifier: str | None, - model: str, - provider: str, provider_config: BaseDecisionsConfig, kwargs: Mapping[str, object], -) -> DecisionsIRRequest: +) -> DecisionsIRRequest | _UnsupportedSafetyIdentifier: ir_request: Final = _ir_request(request, safety_identifier) if ir_request.safety_identifier is None or provider_config.supports_safety_identifier: return ir_request if _drops_params(kwargs): return replace(ir_request, safety_identifier=None) - raise litellm.UnsupportedParamsError( - message=( - f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " - "To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" - ), - model=model, - llm_provider=provider, - ) + return _UnsupportedSafetyIdentifier() def _prepare_call( @@ -213,11 +209,19 @@ def _prepare_call( ir_request: Final = _provider_ir_request( request, safety_identifier=request_safety_identifier, - model=model, - provider=provider, provider_config=provider_config, kwargs=kwargs, ) + if isinstance(ir_request, _UnsupportedSafetyIdentifier): + raise litellm.UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['safety_identifier'], for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + model=model, + llm_provider=provider, + ) body: Final = provider_config.transform_decisions_request( model=canonical_model, request=ir_request, custom_llm_provider=provider )