From e5b2dd52beba4f95d8e084185f18b97b06866c82 Mon Sep 17 00:00:00 2001 From: "Thomas (toto) Bille" Date: Thu, 24 Sep 2026 12:41:41 +0200 Subject: [PATCH 1/3] chore(deps): upgrade any-llm-sdk to 1.29.0 1.29.0 resolves every provider get_supported_providers() lists through LLMProvider, so registry-only providers such as ovhcloud stop being offered by the provider catalog and then rejected on save and dispatch. Its gemini extra requires google-genai 2.x, which the lock now carries. The same release moved llamacpp to a registry row whose API key is optional: it now declares LLAMACPP_API_KEY instead of "None", so the declaration no longer marks it keyless. Classify it with the other self-hosted backends that declare a key they do not require, and move that set to the config layer so a bare `llamacpp:` entry stays silent instead of warning that it needs a key. Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 2 +- src/gateway/core/config.py | 13 +++++++++++-- src/gateway/services/provider_kwargs.py | 24 +++++++++--------------- tests/unit/test_provider_instances.py | 20 ++++++++++++-------- uv.lock | 20 ++++++++++---------- 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0d4b4d87e9..ef5b365a4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ version = "0.0.0" description = "otari, an OpenAI-compatible LLM gateway" requires-python = ">=3.13" dependencies = [ - "any-llm-sdk[all]>=1.28.0", + "any-llm-sdk[all]>=1.29.0", # The built-in guardrail catalog (`services/guardrail_catalog.py`) and the # runner that builds an organization's definitions from it # (`services/tenancy/organization_guardrail_runner.py`). The extras are the diff --git a/src/gateway/core/config.py b/src/gateway/core/config.py index b49053cf4c..ec3ebef149 100644 --- a/src/gateway/core/config.py +++ b/src/gateway/core/config.py @@ -247,11 +247,20 @@ def _get_platform_token_from_env() -> str | None: return token or None +# Self-hosted backends any-llm calls without a key although each declares a +# credential variable: each tolerates a missing key in ``_verify_and_set_api_key`` +# and defaults to a localhost or LAN base URL, so a bare ``vllm:my-model`` reaches +# a local server with nothing configured. The declaration alone cannot tell them +# from a keyed provider, so they are listed by hand and drift-guarded in +# ``tests/unit/test_provider_instances.py``. +KEYLESS_SELF_HOSTED_PROVIDERS = frozenset({"cascadia", "llamacpp", "lmstudio", "otari", "vllm"}) + + def provider_credential_env_names(provider_type: str) -> tuple[str, ...] | None: """Environment variables any-llm reads for a provider's credential. Returns an empty tuple when the provider needs no API key: the keyless local - backends (ollama, llamacpp, llamafile) declare the literal string ``"None"``, + backends ollama and llamafile declare the literal string ``"None"``, and a provider authenticating through a cloud SDK (Vertex AI) declares an empty name. Returns ``None`` when the provider cannot be inspected at all (not a known implementation, or an optional SDK dependency that is not @@ -1773,7 +1782,7 @@ def _warn_on_uncredentialed_bare_entry(self, instance: str) -> None: env_names = provider_credential_env_names(instance) # Empty: a keyless backend, nothing to warn about. None: a provider we # cannot inspect, so we do not know that a credential is needed. - if not env_names: + if not env_names or instance in KEYLESS_SELF_HOSTED_PROVIDERS: return if any(os.getenv(name) for name in env_names): return diff --git a/src/gateway/services/provider_kwargs.py b/src/gateway/services/provider_kwargs.py index 90d5e485aa..c3f8863cda 100644 --- a/src/gateway/services/provider_kwargs.py +++ b/src/gateway/services/provider_kwargs.py @@ -38,7 +38,7 @@ from any_llm.exceptions import AnyLLMError from gateway.auth.vertex_auth import setup_vertex_environment -from gateway.core.config import GatewayConfig, provider_credential_env_names +from gateway.core.config import KEYLESS_SELF_HOSTED_PROVIDERS, GatewayConfig, provider_credential_env_names from gateway.services.alias_service import resolve_effective_alias from gateway.services.catalog_selectors import resolve_catalog_selector from gateway.services.policy_store import resolve_effective_policy @@ -59,19 +59,13 @@ # tolerance (mozilla-ai/any-llm#1198). _KEYLESS_PLACEHOLDER_API_KEY = "otari-no-key-required" -# Two sets of providers any-llm calls without an otari-visible credential, even -# though each *declares* a credential environment variable. -# ``provider_credential_env_names`` sees only the declaration, so it cannot tell -# them from a keyed provider the way it can ollama/llamacpp/llamafile (which +# Providers any-llm calls without an otari-visible credential, even though each +# *declares* a credential environment variable, so ``provider_credential_env_names`` +# cannot tell them from a keyed provider the way it can ollama/llamafile (which # declare the literal ``"None"``) or Vertex AI (which declares an empty name). -# Both are written out by hand and both are drift-guarded in -# ``tests/unit/test_provider_instances.py``. +# The self-hosted ones are ``KEYLESS_SELF_HOSTED_PROVIDERS``; this set is drift-guarded +# alongside it in ``tests/unit/test_provider_instances.py``. # -# Local and LAN backends that never require a key: each overrides -# ``_verify_and_set_api_key`` to return without raising and defaults to a -# localhost or LAN base URL, so a bare ``vllm:my-model`` reaches a self-hosted -# server today with nothing configured in otari at all. -_KEYLESS_SELF_HOSTED_PROVIDERS = frozenset({"vllm", "lmstudio", "cascadia", "otari"}) # Providers authenticating from cloud SDK credentials this gateway cannot see: # an EC2 instance profile, an SSO session, or an ambient boto3 chain. They are # the same category as Vertex AI's application default credentials, which @@ -154,9 +148,9 @@ def credential_ladder_exhausted(provider: LLMProvider, kwargs: dict[str, Any]) - caller that might serve it from somewhere else. A deployment pointing at its own backends is served upstream of anything reading this. They come in two shapes: those declaring no credential variable at all (the keyless local - backends ollama, llamacpp and llamafile, and Vertex AI, which authenticates + backends ollama and llamafile, and Vertex AI, which authenticates through the cloud SDK), and those declaring one any-llm does not insist on - (``_KEYLESS_SELF_HOSTED_PROVIDERS`` and ``_AMBIENT_CREDENTIAL_PROVIDERS``). + (``KEYLESS_SELF_HOSTED_PROVIDERS`` and ``_AMBIENT_CREDENTIAL_PROVIDERS``). ``provider_credential_env_names`` returns ``None`` rather than ``()`` for a provider it cannot inspect at all, and that stays exhausted: nothing is known @@ -165,7 +159,7 @@ def credential_ladder_exhausted(provider: LLMProvider, kwargs: dict[str, Any]) - """ if _kwargs_carry_a_credential(kwargs): return False - if provider.value in _KEYLESS_SELF_HOSTED_PROVIDERS or provider.value in _AMBIENT_CREDENTIAL_PROVIDERS: + if provider.value in KEYLESS_SELF_HOSTED_PROVIDERS or provider.value in _AMBIENT_CREDENTIAL_PROVIDERS: return False if provider_credential_env_names(provider.value) == (): return False diff --git a/tests/unit/test_provider_instances.py b/tests/unit/test_provider_instances.py index 6887320dfe..38c2eb8846 100644 --- a/tests/unit/test_provider_instances.py +++ b/tests/unit/test_provider_instances.py @@ -10,13 +10,17 @@ from any_llm.exceptions import AnyLLMError from gateway.api.routes.pricing import _candidate_model_keys -from gateway.core.config import GatewayConfig, ModelCapabilityConfig, provider_credential_env_names +from gateway.core.config import ( + KEYLESS_SELF_HOSTED_PROVIDERS, + GatewayConfig, + ModelCapabilityConfig, + provider_credential_env_names, +) from gateway.log_config import logger as gateway_logger from gateway.services.model_capabilities import resolve_capabilities from gateway.services.provider_kwargs import ( _AMBIENT_CREDENTIAL_PROVIDERS, _KEYLESS_PLACEHOLDER_API_KEY, - _KEYLESS_SELF_HOSTED_PROVIDERS, get_provider_kwargs, keyless_placeholder_api_key, normalize_pricing_key, @@ -221,7 +225,7 @@ def test_credential_env_names_splits_alternatives() -> None: def test_credential_env_names_empty_for_keyless_backends() -> None: # any-llm spells "no credential" as the literal string "None". - for keyless in ("ollama", "llamacpp", "llamafile"): + for keyless in ("ollama", "llamafile"): assert provider_credential_env_names(keyless) == () @@ -248,7 +252,7 @@ def test_the_uncredentialed_provider_roster_has_not_drifted() -> None: routes a working self-hosted or IAM-authenticated request to somebody else's fleet. `provider_credential_env_names` cannot answer it, because it sees the *declaration* and these providers declare a variable they do not insist on, - so the two sets are written out by hand in `provider_kwargs.py`. + so the two sets are written out by hand in `config.py` and `provider_kwargs.py`. This is the drift guard for both directions, which is why it asserts on the whole roster rather than only on the names already listed. A provider that @@ -270,17 +274,17 @@ def test_the_uncredentialed_provider_roster_has_not_drifted() -> None: # itself and need no hand-written entry, so they are expected here but not in # either set. declares_nothing = {name for name in overriding if provider_credential_env_names(name) == ()} - classified = _KEYLESS_SELF_HOSTED_PROVIDERS | _AMBIENT_CREDENTIAL_PROVIDERS | _KEYED_DESPITE_OVERRIDING + classified = KEYLESS_SELF_HOSTED_PROVIDERS | _AMBIENT_CREDENTIAL_PROVIDERS | _KEYED_DESPITE_OVERRIDING assert overriding - declares_nothing == classified, ( "any-llm's uncredentialed-provider roster changed. Every provider overriding " - "_verify_and_set_api_key must be classified in provider_kwargs.py " - "(_KEYLESS_SELF_HOSTED_PROVIDERS / _AMBIENT_CREDENTIAL_PROVIDERS) or here " + "_verify_and_set_api_key must be classified in config.py " + "(KEYLESS_SELF_HOSTED_PROVIDERS), provider_kwargs.py (_AMBIENT_CREDENTIAL_PROVIDERS) or here " "(_KEYED_DESPITE_OVERRIDING). Unclassified names are treated as keyed, which is " "the direction that hands a working request to a hosted fleet." ) # And nothing in either set has quietly started demanding a key. - for name in _KEYLESS_SELF_HOSTED_PROVIDERS | _AMBIENT_CREDENTIAL_PROVIDERS: + for name in KEYLESS_SELF_HOSTED_PROVIDERS | _AMBIENT_CREDENTIAL_PROVIDERS: assert provider_credential_env_names(name), f"{name} no longer declares a credential variable" assert name in overriding, f"any-llm now unconditionally requires a credential for {name}" diff --git a/uv.lock b/uv.lock index 0435d32b44..16bf31e222 100644 --- a/uv.lock +++ b/uv.lock @@ -256,7 +256,7 @@ wheels = [ [[package]] name = "any-llm-sdk" -version = "1.28.0" +version = "1.29.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anthropic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -267,9 +267,9 @@ dependencies = [ { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/92/0a/c4fbcc0cd516e28eccafba9cfe5d87b45f8849b54e7be0369c339a4a12a2/any_llm_sdk-1.28.0.tar.gz", hash = "sha256:cf9e27f18647ee92f54def3faed035547f6fb9acaa626794c66edcb48f5c118f", size = 228780, upload-time = "2026-09-18T10:09:26.117Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/38/775d83e20d2cc31e4b4ffa71892e689268f961794a756ec22f850a4838f9/any_llm_sdk-1.29.0.tar.gz", hash = "sha256:35ca9051a2c52281429f0465a01b83e3982a1f6048a20dec69841f27a1f3763f", size = 243498, upload-time = "2026-09-24T10:29:29.565Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/3b/379d3885fc1be7a85061d9a049a6561f06f5b4ab0aa98cf93908aa33d025/any_llm_sdk-1.28.0-py3-none-any.whl", hash = "sha256:4fb5e2c82b9e51be6293a97cf6efd6f3fce79684b97537f4f7926d4c3aa5abde", size = 285107, upload-time = "2026-09-18T10:09:24.572Z" }, + { url = "https://files.pythonhosted.org/packages/48/63/ddc8b03cae7050cd4d0a2d20a69882c64b38a7ab75bbc62defd6728b97f0/any_llm_sdk-1.29.0-py3-none-any.whl", hash = "sha256:acfb524f0cc4bb1a0ef0eaa17ca3829a9bcc56653648121ba69d7f1ce3fa35de", size = 302114, upload-time = "2026-09-24T10:29:27.801Z" }, ] [package.optional-dependencies] @@ -1230,7 +1230,7 @@ requires-dist = [ { name = "aiosqlite", specifier = ">=0.19.0" }, { name = "alembic", specifier = ">=1.13.0" }, { name = "any-guardrail", extras = ["azure-content-safety", "bedrock", "openai", "watsonx"], specifier = ">=0.7.7,<0.8.0" }, - { name = "any-llm-sdk", extras = ["all"], specifier = ">=1.28.0" }, + { name = "any-llm-sdk", extras = ["all"], specifier = ">=1.29.0" }, { name = "apron-auth", specifier = ">=0.15.1,<0.16.0" }, { name = "asyncpg", specifier = ">=0.29.0" }, { name = "bcrypt", specifier = ">=5.0.0" }, @@ -1318,15 +1318,15 @@ wheels = [ [[package]] name = "google-auth" -version = "2.49.1" +version = "2.58.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pyasn1-modules", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/80/6a696a07d3d3b0a92488933532f03dbefa4a24ab80fb231395b9a2a1be77/google_auth-2.49.1.tar.gz", hash = "sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64", size = 333825, upload-time = "2026-03-12T19:30:58.135Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/ca/f398a483ce5aad18ca2f735646e45ccee2439bd94a41a4ad0cfa646bd495/google_auth-2.58.0.tar.gz", hash = "sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd", size = 380018, upload-time = "2026-09-09T20:49:38.043Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/eb/c6c2478d8a8d633460be40e2a8a6f8f429171997a35a96f81d3b680dec83/google_auth-2.49.1-py3-none-any.whl", hash = "sha256:195ebe3dca18eddd1b3db5edc5189b76c13e96f29e73043b923ebcf3f1a860f7", size = 240737, upload-time = "2026-03-12T19:30:53.159Z" }, + { url = "https://files.pythonhosted.org/packages/59/13/477d90d09591b3938b45c4e11f4d8a51291682112cb5efcac961e815d562/google_auth-2.58.0-py3-none-any.whl", hash = "sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d", size = 262404, upload-time = "2026-09-09T20:49:33.951Z" }, ] [package.optional-dependencies] @@ -1382,7 +1382,7 @@ wheels = [ [[package]] name = "google-genai" -version = "1.70.0" +version = "2.25.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -1396,9 +1396,9 @@ dependencies = [ { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "websockets", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/dd/28e4682904b183acbfad3fe6409f13a42f69bb8eab6e882d3bcbea1dde01/google_genai-1.70.0.tar.gz", hash = "sha256:36b67b0fc6f319e08d1f1efd808b790107b1809c8743a05d55dfcf9d9fad7719", size = 519550, upload-time = "2026-04-01T10:52:46.487Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/0a/a3b7856ca840031d4393dbdd97b67fe811b20061315ed68b67b5c85ca80d/google_genai-2.25.0.tar.gz", hash = "sha256:ab603baa5eee0205926ad0f8d7f93e0400df6d67650e99c33e01ab228ea16ad6", size = 699564, upload-time = "2026-09-22T17:23:01.238Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/36/a3/d4564c8a9beaf6a3cef8d70fa6354318572cebfee65db4f01af0d41f45ba/google_genai-1.70.0-py3-none-any.whl", hash = "sha256:b74c24549d8b4208f4c736fd11857374788e1ffffc725de45d706e35c97fceee", size = 760584, upload-time = "2026-04-01T10:52:44.349Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a8/178dbb9d1d6cac721b01592e291146a024bae5ee3224e36569348921dd6c/google_genai-2.25.0-py3-none-any.whl", hash = "sha256:2e8f3a5f76ed00d5ffc3153f6089bc7d3511054eed53ad9cfc6095a7dc59b028", size = 1159631, upload-time = "2026-09-22T17:22:59.291Z" }, ] [[package]] From c962e3811b62c77eabb25f5c94df97587a31b6c6 Mon Sep 17 00:00:00 2001 From: "Thomas (toto) Bille" Date: Thu, 24 Sep 2026 12:52:58 +0200 Subject: [PATCH 2/3] fix(messages): accept the container object any-llm 1.29 allows any-llm 1.29.0 lets a Messages request's container be an object with an id and skills, not only a string, which moves the published OpenAPI spec and the dashboard client. Regenerate both. The managed-credential gate already reads an object's id and refuses anything but auto, so an object cannot reach a shared account either. Widen its annotation and cover the object shapes. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/public/openapi.json | 4 ++++ src/gateway/api/routes/messages.py | 2 +- tests/integration/test_hybrid_mode_messages.py | 14 ++++++++++++-- web/src/client/schema.ts | 4 +++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/public/openapi.json b/docs/public/openapi.json index dd2338a5c2..b6bd7f0d17 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -7298,6 +7298,10 @@ { "type": "string" }, + { + "additionalProperties": true, + "type": "object" + }, { "type": "null" } diff --git a/src/gateway/api/routes/messages.py b/src/gateway/api/routes/messages.py index 4cc99dc5fe..93cf7a6787 100644 --- a/src/gateway/api/routes/messages.py +++ b/src/gateway/api/routes/messages.py @@ -706,7 +706,7 @@ def prepare_platform_call_kwargs(self, kwargs: dict[str, Any]) -> dict[str, Any] ) -def _reject_container_on_managed_credential(ctx: RequestContext, container: str) -> None: +def _reject_container_on_managed_credential(ctx: RequestContext, container: str | dict[str, Any]) -> None: """Refuse a caller-chosen container id when the upstream account is not the caller's. A container id names an execution environment and the files uploaded into it, diff --git a/tests/integration/test_hybrid_mode_messages.py b/tests/integration/test_hybrid_mode_messages.py index 4a0b1a049b..1804db1da2 100644 --- a/tests/integration/test_hybrid_mode_messages.py +++ b/tests/integration/test_hybrid_mode_messages.py @@ -1254,12 +1254,22 @@ async def fake_amessages(**kwargs: Any) -> MessageResponse: monkeypatch.setattr("gateway.api.routes.messages.amessages", fake_amessages) +@pytest.mark.parametrize( + "container", + [ + "container_01ABC", + {"id": "container_01ABC"}, + {"skills": [{"type": "anthropic", "skill_id": "pptx"}]}, + ], + ids=["id", "object-with-id", "object-with-skills-only"], +) def test_container_is_refused_on_a_managed_credential( platform_client: TestClient, monkeypatch: pytest.MonkeyPatch, + container: str | dict[str, Any], ) -> None: """A managed attempt runs on an account many workspaces share, so a - caller-chosen container id would address another tenant's state.""" + caller-chosen container would address or create state on that account.""" calls: list[str] = [] _container_route(monkeypatch, managed=True, calls=calls) @@ -1269,7 +1279,7 @@ def test_container_is_refused_on_a_managed_credential( "model": "claude-3-5-sonnet-20241022", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100, - "container": "container_01ABC", + "container": container, }, headers={"Authorization": "Bearer user_test_token"}, ) diff --git a/web/src/client/schema.ts b/web/src/client/schema.ts index ee32da096a..5a60a29d45 100644 --- a/web/src/client/schema.ts +++ b/web/src/client/schema.ts @@ -8907,7 +8907,9 @@ export interface components { [key: string]: unknown; } | null; /** Container */ - container?: string | null; + container?: string | { + [key: string]: unknown; + } | null; /** Context Management */ context_management?: { [key: string]: unknown; From 19282f8003c6b87cf03a4b0700178af3978ec8d2 Mon Sep 17 00:00:00 2001 From: "Thomas (toto) Bille" Date: Thu, 24 Sep 2026 13:35:17 +0200 Subject: [PATCH 3/3] test(messages): expect the container object in the published contract The regenerated spec advertises container as a string or an object, so the contract test pinned to the string form fails. Assert the new shape and cover the object surviving request validation. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/unit/test_request_schema_derivation.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_request_schema_derivation.py b/tests/unit/test_request_schema_derivation.py index a1228e6a50..cdf1c093f5 100644 --- a/tests/unit/test_request_schema_derivation.py +++ b/tests/unit/test_request_schema_derivation.py @@ -114,13 +114,28 @@ def test_messages_request_preserves_container_for_upstream() -> None: assert request.model_dump(exclude_unset=True)["container"] == "container_01ABC" +def test_messages_request_preserves_a_container_object_for_upstream() -> None: + """Anthropic's object form, an id and the skills to load, survives validation unchanged.""" + container = {"id": "container_01ABC", "skills": [{"type": "anthropic", "skill_id": "pptx"}]} + request = MessagesRequest.model_validate( + { + "model": "anthropic:claude-sonnet-4-5", + "messages": [{"role": "user", "content": "Continue"}], + "max_tokens": 100, + "container": container, + } + ) + + assert request.model_dump(exclude_unset=True)["container"] == container + + def test_messages_openapi_request_schema_includes_container() -> None: - """The published Messages request contract advertises container continuity.""" + """The published Messages request contract advertises container continuity, as an id or an object.""" spec_path = Path(__file__).resolve().parents[2] / "docs/public/openapi.json" spec = json.loads(spec_path.read_text()) container = spec["components"]["schemas"]["MessagesRequest"]["properties"]["container"] - assert {variant.get("type") for variant in container["anyOf"]} == {"string", "null"} + assert {variant.get("type") for variant in container["anyOf"]} == {"string", "object", "null"} def test_derived_and_hand_written_endpoints_are_disjoint() -> None: