Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions docs/run-inference/about.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,33 @@ working as designed — use `/health/ready` instead.

---

## Troubleshooting

### Function tools require disabled reasoning

If an OpenAI-compatible Chat Completions request using function tools fails
with an error like this:

```text
Function tools with reasoning_effort are not supported for <model> in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.
```

set `reasoning_effort` in the affected provider's default request body. Replace
`<provider_name>` and `<workspace>` with the provider and workspace used for
the request:

```bash
nemo inference providers update <provider_name> \
--workspace <workspace> \
--default-extra-body '{"reasoning_effort":"none"}'
```

Provider requirements differ. Configure only the parameters required by the
provider and model you use; `--default-extra-body` applies them to requests
routed through that provider and can be overridden by an individual request.
Comment thread
coderabbitai[bot] marked this conversation as resolved.

---

## API Reference

For complete API details, refer to the [Inference Gateway API Reference](/documentation/reference/api-reference) and [SDK Reference](/documentation/reference/python-sdk).
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from contextvars import ContextVar
from dataclasses import dataclass

from litellm import get_model_info
from nemo_platform import AsyncNeMoPlatform
from nemo_platform.config import get_context
from nemo_platform.types.inference import ModelProvider
Expand All @@ -24,26 +23,10 @@
_PLACEHOLDER_API_KEY = "not-needed"
_OPENAI_FORMAT = "OPENAI_CHAT"
_ANTHROPIC_FORMAT = "ANTHROPIC_MESSAGES"
_OPENAI_CHAT_REASONING_EFFORT = "none"
_ACCEPT_ENCODING_HEADER = "accept-encoding"
_IDENTITY_ENCODING = "identity"


def _openai_chat_reasoning_effort(model: str) -> str | None:
"""Disable reasoning unless LiteLLM explicitly says this value is unsupported."""
try:
model_info = get_model_info(model)
except Exception as exc:
# Custom provider registrations need not appear in LiteLLM's model map.
# Preserve the value that makes frontier Chat Completions tool calls work.
if "This model isn't mapped yet" not in str(exc):
raise
return _OPENAI_CHAT_REASONING_EFFORT
if model_info.get("supports_none_reasoning_effort") is False:
return None
return _OPENAI_CHAT_REASONING_EFFORT


@dataclass(frozen=True)
class ConfiguredModelRefs:
"""Workspace-qualified Model Entity IDs for default and fast agent work."""
Expand Down Expand Up @@ -131,10 +114,6 @@ def _completion_client(
# LiteLLM versions otherwise bridge GPT-5.4+ tool calls with any
# reasoning_effort value (including "none") to /responses.
_skip_responses_api_bridge=True,
# Chat Completions tool calls on current OpenAI frontier models
# require reasoning to be disabled. Use LiteLLM's capability map to
# omit that value only when the served model explicitly rejects it.
reasoning_effort=_openai_chat_reasoning_effort(litellm_model),
)
elif model_entity.backend_format == _ANTHROPIC_FORMAT:
api_base = api_base.removesuffix("/v1")
Expand Down
32 changes: 0 additions & 32 deletions packages/nemo_platform_plugin/tests/test_nooa_model_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from nemo_platform_plugin.nooa_model_client import (
ConfiguredModelClients,
ConfiguredModelRefs,
_openai_chat_reasoning_effort,
activate_model_clients,
configured_model_refs,
get_configured_model_refs,
Expand Down Expand Up @@ -76,7 +75,6 @@ async def test_resolve_model_clients_deduplicates_same_model(monkeypatch):
extra_headers={"x-test": "value", "accept-encoding": "identity"},
drop_params=True,
_skip_responses_api_bridge=True,
reasoning_effort="none",
)
assert default_headers == {"x-test": "value"}

Expand Down Expand Up @@ -177,7 +175,6 @@ async def test_resolve_model_clients_uses_provider_served_name(monkeypatch):
extra_headers={"accept-encoding": "identity"},
drop_params=True,
_skip_responses_api_bridge=True,
reasoning_effort="none",
)


Expand Down Expand Up @@ -251,32 +248,3 @@ async def test_resolve_model_clients_closes_constructed_client_after_failure(mon
)

constructed.aclose.assert_awaited_once()


def test_openai_chat_reasoning_effort_uses_litellm_capability(monkeypatch):
monkeypatch.setattr(
nooa_model_client,
"get_model_info",
lambda model: {"supports_none_reasoning_effort": False},
)

assert _openai_chat_reasoning_effort("openai/gpt-5-mini") is None


def test_openai_chat_reasoning_effort_preserves_none_for_unmapped_models(monkeypatch):
def unmapped(model: str):
raise Exception("This model isn't mapped yet")

monkeypatch.setattr(nooa_model_client, "get_model_info", unmapped)

assert _openai_chat_reasoning_effort("openai/custom-model") == "none"


def test_openai_chat_reasoning_effort_does_not_hide_lookup_failures(monkeypatch):
def failed(model: str):
raise RuntimeError("model registry failed")

monkeypatch.setattr(nooa_model_client, "get_model_info", failed)

with pytest.raises(RuntimeError, match="model registry failed"):
_openai_chat_reasoning_effort("openai/gpt-5-mini")
Loading