diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 575a1d8a1..948d505f2 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -312,8 +312,8 @@ def main() -> None: "--temperature", dest="sampling_temperature", type=float, - default=0.2, - help="Default provider sampling temperature (default: 0.2; --temperature is a compatibility alias).", + default=None, + help="Optional provider sampling temperature; omitted by default so model capabilities decide (the alias is retained for compatibility).", ) parser.add_argument("--max-output-tokens", type=int, default=2048, help="Default provider output token cap (default: 2048).") diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 3f0b5e8eb..bf61e5542 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -178,13 +178,16 @@ def complete_structured( if not isinstance(response_format, dict): raise TypeError("response_format must be a mapping") agent = self._agent() - response = self.orchestrator.proxy_completion({ + payload: dict[str, Any] = { "model": agent.model, "messages": messages, - "temperature": self.orchestrator.client.temperature, - "max_tokens": self.orchestrator.client.max_output_tokens, + "max_tokens": self.orchestrator.client._effective_max_output_tokens(), "response_format": response_format, - }) + } + effective_temperature = self.orchestrator.client._effective_temperature() + if effective_temperature is not None: + payload["temperature"] = effective_temperature + response = self.orchestrator.proxy_completion(payload) output = ModelClient._response_content(agent, response) usage = response.get("usage") if isinstance(response.get("usage"), dict) else None return self._completion_payload(output, agent.id, usage, self.mode if mode is None else mode) @@ -735,7 +738,7 @@ def __init__( local_max_retries: int = 0, retry_backoff: float = 0.5, retry_backoff_cap: float = 8.0, - temperature: float = 0.2, + temperature: float | None = None, local_concurrency: int = 1, chat_template_args: dict[str, Any] | None = None, ca_bundle: str | None = None, @@ -746,7 +749,9 @@ def __init__( self.max_output_tokens = max_output_tokens if isinstance(max_retries, bool) or max_retries < 0: raise ValueError("max_retries must be >= 0") - self.default_temperature = 0.2 + # Do not invent a sampling parameter for a provider/model that may not + # support it. Explicit caller values are still forwarded after validation. + self.default_temperature = temperature self.default_top_p: float | None = None self.default_presence_penalty: float | None = None self.default_frequency_penalty: float | None = None @@ -756,7 +761,6 @@ def __init__( self.local_max_retries = int(local_max_retries) self.retry_backoff = retry_backoff self.retry_backoff_cap = retry_backoff_cap - self.temperature = temperature if type(local_concurrency) is not int or not 1 <= local_concurrency <= MAX_LOCAL_CONCURRENCY: raise ValueError( f"local_concurrency must be an integer in 1..{MAX_LOCAL_CONCURRENCY}" @@ -812,6 +816,48 @@ def take_usage(self) -> dict[str, Any] | None: self._local.usage = None return usage + @contextmanager + def request_options( + self, + *, + max_output_tokens: int | None = None, + temperature: float | None = None, + top_p: float | None = None, + presence_penalty: float | None = None, + frequency_penalty: float | None = None, + ): + """Apply HTTP request controls without mutating shared client defaults.""" + missing = object() + options = { + "request_max_output_tokens": max_output_tokens, + "request_temperature": temperature, + "request_top_p": top_p, + "request_presence_penalty": presence_penalty, + "request_frequency_penalty": frequency_penalty, + } + previous = {name: getattr(self._local, name, missing) for name in options} + for name, value in options.items(): + if value is not None: + setattr(self._local, name, value) + try: + yield + finally: + for name, value in previous.items(): + if value is missing: + self._local.__dict__.pop(name, None) + else: + setattr(self._local, name, value) + + def _effective_temperature(self, requested: float | None = None) -> float | None: + """Resolve request, request-scoped, and constructor sampling values in order.""" + if requested is not None: + return requested + return getattr(self._local, "request_temperature", self.default_temperature) + + def _effective_max_output_tokens(self) -> int: + """Resolve the request-scoped output limit or the constructor default.""" + return getattr(self._local, "request_max_output_tokens", self.max_output_tokens) + def chat( self, agent: ModelAgent, @@ -821,16 +867,24 @@ def chat( ) -> str: """Send messages to a mock or OpenAI-compatible chat endpoint with retries. - When ``temperature``/``top_p`` are omitted, ``default_temperature`` and - ``default_top_p`` are used so request-scoped Completions sampling can be - applied without threading kwargs through every orchestrator hop. + When sampling knobs are omitted, they are omitted from the provider + request so the selected model's capability contract remains authoritative. + Explicit request-scoped values are still forwarded after validation. """ self._local.usage = None # Expose the effective sampling knobs for request-path tests / diagnostics. - effective_temperature = self.default_temperature if temperature is None else temperature - effective_top_p = self.default_top_p if top_p is None else top_p - effective_presence = self.default_presence_penalty - effective_frequency = self.default_frequency_penalty + effective_temperature = self._effective_temperature(temperature) + effective_top_p = ( + getattr(self._local, "request_top_p", self.default_top_p) + if top_p is None + else top_p + ) + effective_presence = getattr( + self._local, "request_presence_penalty", self.default_presence_penalty + ) + effective_frequency = getattr( + self._local, "request_frequency_penalty", self.default_frequency_penalty + ) self._local.last_temperature = effective_temperature self._local.last_top_p = effective_top_p self._local.last_presence_penalty = effective_presence @@ -849,10 +903,11 @@ def chat( payload = { # pragma: no cover "model": agent.model, "messages": messages, - "temperature": effective_temperature, "stream": False, - "max_tokens": self.max_output_tokens, + "max_tokens": self._effective_max_output_tokens(), } + if effective_temperature is not None: # pragma: no cover + payload["temperature"] = effective_temperature if effective_top_p is not None: # pragma: no cover payload["top_p"] = effective_top_p if effective_presence is not None: # pragma: no cover @@ -906,7 +961,6 @@ def probe(self, agent: ModelAgent, *, timeout: float = DEFAULT_PROVIDER_PROBE_TI payload: dict[str, Any] = { "model": agent.model, "messages": [{"role": "user", "content": "Reply with exactly OK."}], - "temperature": 0.0, "stream": False, "max_tokens": 1, } @@ -1132,10 +1186,25 @@ def stream_chat(self, agent: ModelAgent, messages: list[ChatMessage], temperatur payload = { # pragma: no cover "model": agent.model, "messages": messages, - "temperature": self.temperature if temperature is None else temperature, "stream": True, - "max_tokens": self.max_output_tokens, + "max_tokens": self._effective_max_output_tokens(), } + effective_temperature = self._effective_temperature(temperature) + if effective_temperature is not None: # pragma: no cover + payload["temperature"] = effective_temperature + effective_top_p = getattr(self._local, "request_top_p", self.default_top_p) + if effective_top_p is not None: # pragma: no cover + payload["top_p"] = effective_top_p + effective_presence = getattr( + self._local, "request_presence_penalty", self.default_presence_penalty + ) + if effective_presence is not None: # pragma: no cover + payload["presence_penalty"] = effective_presence + effective_frequency = getattr( + self._local, "request_frequency_penalty", self.default_frequency_penalty + ) + if effective_frequency is not None: # pragma: no cover + payload["frequency_penalty"] = effective_frequency if _is_direct_mlx_provider_url(agent.base_url) and self.chat_template_args: payload["chat_template_kwargs"] = self.chat_template_args with _local_provider_slot(agent, self.local_concurrency, self.timeout): # pragma: no cover @@ -1410,26 +1479,28 @@ def _batch_run( self, agent: ModelAgent, requests: dict[str, list[ChatMessage]], - temperature: float, + temperature: float | None, poll_interval: float, poll_timeout: float, destination: ProviderDestination | None = None, ) -> dict[str, dict[str, Any]]: """Upload, create, poll, and parse one batch (isolated so the flow stays testable).""" - lines = [ - json.dumps({ + lines = [] + for custom_id, messages in requests.items(): + body: dict[str, Any] = { + "model": agent.model, + "messages": messages, + "max_tokens": self._effective_max_output_tokens(), + } + effective_temperature = self._effective_temperature(temperature) + if effective_temperature is not None: + body["temperature"] = effective_temperature + lines.append(json.dumps({ "custom_id": custom_id, "method": "POST", "url": "/v1/chat/completions", - "body": { - "model": agent.model, - "messages": messages, - "temperature": self.temperature if temperature is None else temperature, - "max_tokens": self.max_output_tokens, - }, - }, ensure_ascii=False) - for custom_id, messages in requests.items() - ] + "body": body, + }, ensure_ascii=False)) input_file_id = self._batch_upload(agent, "\n".join(lines).encode("utf-8"), destination) batch_id = self._batch_json(agent, "POST", "/batches", { "input_file_id": input_file_id, diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 97390eb39..5dcb97209 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -23,6 +23,7 @@ BudgetExceededError, MAX_LOCAL_CONCURRENCY, TaskOrchestrator, + _is_local_provider_url, _new_chat_completion_id, chat_completion_chunks, chat_completion_response, @@ -4917,6 +4918,12 @@ def do_POST(self) -> None: # noqa: N802 if len(segments) != 5 or segments[:3] != ["api", "v1", "agent_pools"]: raise RequestError(400, "bad_path", "agent create path must be /api/v1/agent_pools/{pool}/worker_agents") _reject_unknown_keys(body, ALLOWED_AGENT_CREATE_KEYS) + if _is_local_provider_url(str(body.get("base_url", ""))): + raise RequestError( + 400, + "local_provider_registration_forbidden", + "Configure local providers at trusted process startup instead.", + ) self._send(orchestrator.add_agent(segments[3], body), 201) return @@ -4980,24 +4987,14 @@ def do_POST(self) -> None: # noqa: N802 attribution["service"] = "completions_api" routing = _validate_routing(body.get("routing")) started_at = time.perf_counter() - # Apply request sampling knobs to the provider client for this call. model_client = orchestrator.client - previous_max_tokens = model_client.max_output_tokens - previous_temperature = model_client.default_temperature - previous_top_p = model_client.default_top_p - previous_presence = model_client.default_presence_penalty - previous_frequency = model_client.default_frequency_penalty - if max_tokens is not None: - model_client.max_output_tokens = max_tokens - if temperature is not None: - model_client.default_temperature = temperature - if top_p is not None: - model_client.default_top_p = top_p - if presence_penalty is not None: - model_client.default_presence_penalty = presence_penalty - if frequency_penalty is not None: - model_client.default_frequency_penalty = frequency_penalty - try: + with model_client.request_options( + max_output_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + ): result = self._run(lambda: coordinator.complete( messages, mode="route", @@ -5006,12 +5003,6 @@ def do_POST(self) -> None: # noqa: N802 model_name=model_name, workflow_run_id=f"run_{uuid.uuid4().hex}", )) - finally: - model_client.max_output_tokens = previous_max_tokens - model_client.default_temperature = previous_temperature - model_client.default_top_p = previous_top_p - model_client.default_presence_penalty = previous_presence - model_client.default_frequency_penalty = previous_frequency # Batch-channel Completions return a job handle (202), not a # text_completion body — match chat Completions honesty so # clients never receive a 500 on a valid batch routing hint. @@ -5226,22 +5217,13 @@ def do_POST(self) -> None: # noqa: N802 _validate_openai_metadata(body) started_at = time.perf_counter() model_client = orchestrator.client - previous_max_tokens = model_client.max_output_tokens - previous_temperature = model_client.default_temperature - previous_top_p = model_client.default_top_p - previous_presence = model_client.default_presence_penalty - previous_frequency = model_client.default_frequency_penalty - if max_tokens is not None: - model_client.max_output_tokens = max_tokens - if temperature is not None: - model_client.default_temperature = temperature - if top_p is not None: - model_client.default_top_p = top_p - if presence_penalty is not None: - model_client.default_presence_penalty = presence_penalty - if frequency_penalty is not None: - model_client.default_frequency_penalty = frequency_penalty - try: + with model_client.request_options( + max_output_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + ): if stream and orchestrator.would_route(messages, mode): self._stream_route_completion(orchestrator, security, messages, model_name) orchestrator.record_analytics_event( @@ -5264,12 +5246,6 @@ def do_POST(self) -> None: # noqa: N802 model_name=model_name, workflow_run_id=f"run_{uuid.uuid4().hex}", )) - finally: - model_client.max_output_tokens = previous_max_tokens - model_client.default_temperature = previous_temperature - model_client.default_top_p = previous_top_p - model_client.default_presence_penalty = previous_presence - model_client.default_frequency_penalty = previous_frequency # Latency-tolerant requests get dispatched to the batch backend. if result.get("channel") == "batch": orchestrator.record_analytics_event( diff --git a/docs/adr/0123-omit-unsupported-sampling-defaults.md b/docs/adr/0123-omit-unsupported-sampling-defaults.md new file mode 100644 index 000000000..d51cdce4f --- /dev/null +++ b/docs/adr/0123-omit-unsupported-sampling-defaults.md @@ -0,0 +1,51 @@ +# ADR 0123: Omit Unsupported Sampling Defaults + +- Status: Accepted +- Date: 2026-08-21 + +## Context + +Reasoning-capable providers can reject a `temperature` field even when the +value is a conventional default such as `0.2`. A gateway that invents this +field before capability negotiation makes an otherwise valid discovered model +unusable and can turn failover into a false provider outage. Azure's current +reasoning-model documentation explicitly lists `temperature` among the +unsupported parameters (Microsoft, 2026). + +## Decision + +The orchestrator omits `temperature` and other optional sampling controls when +the caller did not explicitly provide them. The provider and selected model +therefore own their documented defaults and capability handling. Explicit +values remain validated at the public request boundary and are forwarded +unchanged through chat, streaming, batch, and structured-output transport +paths. A client-level explicit value, including the CLI option, applies to +ordinary chat as well as streaming and batch calls; a request-scoped value +temporarily overrides it for both streamed and non-streamed requests. HTTP +request overrides are isolated from concurrent requests; the server must not +mutate shared client defaults while a request is in flight. + +The command-line sampling option is optional and defaults to omission. Health +probes also omit sampling controls; a probe must test reachability without +assuming that the model accepts a sampling parameter. + +This is a transport-capability contract, not a model-ranking heuristic. Model +selection and reasoning-effort policy remain governed by the paper-grounded +model policy ADRs. + +## Consequences + +- Reasoning models that reject `temperature` can be selected without a local + model-name exception. +- Callers that require a sampling value must state it explicitly and accept the + selected provider's capability response. +- One request's explicit controls cannot become another request's implicit + defaults. +- Provider request assertions must distinguish omitted fields from explicit + values. + +## References + +Microsoft. (2026, August 6). *Azure OpenAI reasoning models: GPT-5 series, +o3-mini, o1, o1-mini*. Microsoft Learn. +https://learn.microsoft.com/azure/foundry/openai/how-to/reasoning diff --git a/docs/adr/0124-local-provider-registration-boundary.md b/docs/adr/0124-local-provider-registration-boundary.md new file mode 100644 index 000000000..19382f8f1 --- /dev/null +++ b/docs/adr/0124-local-provider-registration-boundary.md @@ -0,0 +1,61 @@ +# ADR 0124: Keep local providers out of HTTP agent registration + +- Status: Accepted +- Date: 2026-08-22 + +## Context + +Local `mlx://` and `local://` providers intentionally connect to a loopback +model runtime whose port is chosen by the operator. The HTTP agent-pool API +also accepted those URLs. Consequently, an actor holding the administrative +API credential could persist a worker that made the orchestrator connect to an +unrelated loopback service. Strix reported that path as server-side request +forgery (SSRF). + +The transport already resolves every endpoint before use, rejects non-loopback +addresses for local providers, rejects non-public addresses for remote +providers, and connects to the validated address without a DNS relookup. Those +controls prevent remote-host and DNS-pinning escapes, but they do not establish +that an arbitrary loopback port is a model runtime. + +## Decision + +The HTTP agent-pool create endpoint rejects `mlx://` and `local://` provider +URLs before an agent can be persisted. Local providers remain available through +trusted process-startup configuration, where the operator controls the runtime +and port. HTTP-created remote providers retain the existing HTTPS, credential, +host-policy, public-address, and pinned-destination checks. + +We do not use a generic loopback-port allowlist. A port number is not a service +identity, local model runtimes legitimately use configurable ports, and an +unrelated process could bind an otherwise allowed port. Separating trusted +bootstrap configuration from the remotely mutable control plane closes the +confused-deputy path with the smaller and stronger boundary. + +## Consequences + +- Buyers can still run MLX and other local models by declaring them at trusted + startup. +- API clients receive a named `400` response directing them to startup + configuration when they try to register a local provider. +- Even an authenticated admin request cannot turn the orchestrator into a + loopback port scanner through dynamic agent creation. +- Existing mock agents and HTTPS provider registration remain unchanged. + +## Standards and research grounding + +OWASP recommends allowlisting identified trusted destinations and applying +defense in depth for SSRF. NIST zero-trust guidance requires least-privilege, +per-request access decisions instead of granting broad trust after +authentication. The selected boundary applies both principles without treating +an arbitrary local port as an identity. + +## References + +Open Worldwide Application Security Project. (n.d.). *Server-side request +forgery prevention cheat sheet*. OWASP Cheat Sheet Series. Retrieved August 22, +2026, from https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html + +Rose, S., Borchert, O., Mitchell, S., & Connelly, S. (2020). *Zero trust +architecture* (NIST Special Publication 800-207). National Institute of +Standards and Technology. https://doi.org/10.6028/NIST.SP.800-207 diff --git a/pyproject.toml b/pyproject.toml index 5471e0017..bd32d2456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ db = [ "SQLAlchemy>=2.0", "alembic>=1.17", "psycopg[binary]>=3.2", + "greenlet>=1", ] fuzz = [ "atheris==3.1.0; python_version >= '3.12'", diff --git a/requirements.lock b/requirements.lock index dce5c956a..bc3e0e29f 100644 --- a/requirements.lock +++ b/requirements.lock @@ -112,11 +112,94 @@ greenlet==3.5.3 \ --hash=sha256:efc6bd60ea02e085862c74a3ef64b147ffc6f1a5ea7d9f26e7a939943f68c1e3 \ --hash=sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154 \ --hash=sha256:fd2e02fa07485778536a036222d616ab957b1d533f36b3ed98ce725d9c9d3117 - # via sqlalchemy + # via contextual-orchestrator (pyproject.toml) h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via uvicorn +hypothesis==6.165.10 \ + --hash=sha256:00de0abdcf8c05c9d0eab735a3c49a276376b55151e6fcb903c2b39a90e5e5c3 \ + --hash=sha256:057d0232f1224dcd0b7698902551a4341a7399f90670b036db6c4376715fe889 \ + --hash=sha256:09772e328a26e50486ac572be34f9887f9aa185efe7ebb16bde4e8f6038db1f4 \ + --hash=sha256:0c4e6869817c3cfdf5a2b4d348497b95159bdecb3365be732c9b8570e36a4eef \ + --hash=sha256:10d9a650a4666b0914831f769703d36140ed8039fd19bf9b71f615b8541eccf2 \ + --hash=sha256:18a3ea838ddea183388f8788750afa8494d79abb5358823be9782585f34445d3 \ + --hash=sha256:1a380bc99aa3b035e6a95a2201bf792d4082a04ca75babcc21849c2d0914bb28 \ + --hash=sha256:1d305448e9bd8e2f4f3cea0eafd809efdaab4e998a0019bc615650c8463e42f1 \ + --hash=sha256:1ec53f08732e3cfd0342cbbd75dbd1b193c8f19390660466e536a748bb81f757 \ + --hash=sha256:1f2c4db25fb8ec1a16a8dba580666337b8ffb1887c4cf1750cc954313897cef7 \ + --hash=sha256:20f6236cfb90b7817bb1a6a087589ca4aa46d73170f0dd62963952ed5dadc589 \ + --hash=sha256:22cf19388f0ff6ced8eb3e49c903d14938e4ed909d93bf28383eef451511e424 \ + --hash=sha256:277f41801e88dad2eba082f91a75632b7584ff64044ba2cf9dadf511b0d19cd0 \ + --hash=sha256:2a2567b3a03a4a5a7c575c191cfcce321a967df3727803817e75bffbbeaecabe \ + --hash=sha256:2abb50cf1cf77d721de0a24c3f99d9c4ffdeb2cbd1e12aebb5a7a93e2b6b6d1f \ + --hash=sha256:2b112768cfb67f2b683e53e58c1a33d27811aacf60c942b8eb74635e469a73f6 \ + --hash=sha256:2b36aaffc88625a44f91074c5bbedfdefb9b376c38d1b3c342edcd2e4c8ed16c \ + --hash=sha256:2d0e0f8263d34dd8fa3b39eaa9a50bba56a8470b3dd9ebf6672d10840abe063e \ + --hash=sha256:30797f20ca45e57f526d2df872f63ba453cb4e1091ad542184a7a951af8da79d \ + --hash=sha256:3376f2594763aef14faa519b0fb27cae7ce9eeaab4c69efa07777499110306c9 \ + --hash=sha256:34ee6402df6f31274d89119f1561b5f7489c97866afc5b7a3ed3a13d7e762802 \ + --hash=sha256:37a7ac3d34220800e1107871cc391bca1b00439875925d7d821878b8b791f245 \ + --hash=sha256:3de69aa8b924b400291a3cc42aaf78e6ab65c905a3e7e1a5dc39d95ef1b428cb \ + --hash=sha256:4334058033e0214475f019e15492a50f3854fe8728cf51fe25c6191a2c3f8e52 \ + --hash=sha256:490c56b830772b0eca3b4b2cecb3741a1ed26b1d7206a279e1525dbf0aa95ee4 \ + --hash=sha256:4c68e983d0007d014bb01ad4bcbba78bc432c73a1755ff36d5102ceefa18299a \ + --hash=sha256:5671d2b2bf83bd4b6f02e55b32d432506eff5358c82f39b460a849ce19a2666e \ + --hash=sha256:56cb8c9055e50545fe6e3e5a560ec25a724673b2e4051f3c24d44e3ebc35dd72 \ + --hash=sha256:5841331c504e02d7c334591681cb8587cdd59dee7e149db6d3db8e3f9e9f02eb \ + --hash=sha256:592107a0faf6c9c3a63a8dbf13dfb1cbda1cf599b0bc11c953221b00204b9ce1 \ + --hash=sha256:5cf3b612542ba174c9da4000b59a4f4c81e8d66f87509be85d3a1b71b5c36413 \ + --hash=sha256:60cab3ab4ea468d31a33739ffd7e94ec3e37dea891d65a6582ecc8a477175191 \ + --hash=sha256:637445c1593a2a9d1024fda50082f07bb56baedda78d90a25f64b8111727ef94 \ + --hash=sha256:68b45e09834cd80523cb1eb274463073c7a9af4e4ef7cff34d9615f355572d32 \ + --hash=sha256:6caadcd1afb62630ff5c5ff353626eaa616553a5971295ad6dc2b19ca8a39620 \ + --hash=sha256:6e20a02775eb3cf0ffb4f0219b6d7c1f240336663d4e5d7028675ec247c790c4 \ + --hash=sha256:713f4ce4e82c26b53031f139de959bc9e8b54d3995aa824b89bbdf8229df2a45 \ + --hash=sha256:717aea574e0e5edba2868aa66b1caae335d8f1ad3fb29f01dd6502953fa823a1 \ + --hash=sha256:72df95fb1db41755b155c5f02106e0036a339250555c8d351d488704fd112cf9 \ + --hash=sha256:73e6df02a6a62f8045b511c272f894d08e56d174504c793c9effcbc6778051a8 \ + --hash=sha256:76a7be86d986223b9f1bdb7e7cbcdb048649901fdb956c598ef73bdab1786cd5 \ + --hash=sha256:7730d8197086f65d8969a991d6728a1d420a51b19fea06535c896cb43a1e05d0 \ + --hash=sha256:79900a9920a0b1d3a626c03a90ac6bf7042e78d46906a565b86a0dbe926f1d96 \ + --hash=sha256:7a7980a898a3e6ebe4de1896a0507e3d519edb53fb9b4bda478c9fbeb6514558 \ + --hash=sha256:8001925fa3dde51cb574e4c9de4c7efe77c4e4d64bd2fd2ef61d5651f9d04f3d \ + --hash=sha256:8660572b2d424bf5369ea8990985225f70bd1615b76ecd9c25588a3b9307009f \ + --hash=sha256:8b20f44773a9ab84400465e318712d8c2ca16418d35b9f80aa27fdf2d690ad10 \ + --hash=sha256:90915635b9648071129b0f72c0673cf8eac9eb84cfd445c5bedef30c714b1ec2 \ + --hash=sha256:9ccac776b2ca93b324806facd526ccb45da0fd035001c899a35b02c44431e209 \ + --hash=sha256:9d77c3be7b429875036ad0f0597c6e5cc6bb17894a4da005e3807de64d2673ad \ + --hash=sha256:9f07ae36c3b093e13687a894e79fe69e98a94c0b67fef656c575247682218143 \ + --hash=sha256:ab0f2e9d7d7d4db257f7cf53de3706c2baf124269571f20ffc2bcd6781f03063 \ + --hash=sha256:ad0764730e8e3421601c2cc7e1f054a9206c60ea0917165d8d9193dc453f34f1 \ + --hash=sha256:aff1f584c9538e8979cd180b1d70bf99bc16be19d4666414f49e5942b21a4f2c \ + --hash=sha256:b33dc30170a7402e03c180f2c5ef69dc077152f35b91621e9cebcde9c7d71746 \ + --hash=sha256:b5820d009aedb7ae9cfd32f98b1ab0c0bbd6268379c4fab042218b6b655c63f8 \ + --hash=sha256:bb8c7d05ea27a093a92b250904095d71d924b6b44e5795a415c1b20c265f0c65 \ + --hash=sha256:c01dd04044c472e47193b54f68e84e08d6ebf4f29551885aa959b015f7cd9747 \ + --hash=sha256:c53e9b1c36350df9965ec44d6c0d4e0bbbb38f720dd2b0e1256dc6524d411015 \ + --hash=sha256:c6559380469295c4009215fe1cab561301591a3bee2e2fb3f4f96d2273a3affc \ + --hash=sha256:cc2da5aa4edf14743fa9257e5ba3513963999f01211635702479d8e92b8207c8 \ + --hash=sha256:d1ea02fa8ab3d33eb1125eade81f7136341eb429152c6dbe2ae6f8bc33b3fbdd \ + --hash=sha256:d623801ae3dcd97b77b983400ef3d48bf976648e4efff19929175322eaae074d \ + --hash=sha256:d9145fe43ebb22e66672967c3fab411793b226ed776e4fe282271bca6ad3c0bb \ + --hash=sha256:dafa7c9dbe3d802f9bcdf261b29c8a70700fb22839947f06e471f62c46b6257f \ + --hash=sha256:dd207497bb985918409a1bb5db85d1875f74e1269487332113b73d1ee7c77647 \ + --hash=sha256:e10858f57ed0e74baa04393845f469fe8ad502c16ece4499bef7700c575611bd \ + --hash=sha256:e1bbeb7c506b07ee0422cf9b2f7212fefa4240957f03526d38d27bc6743a0a48 \ + --hash=sha256:e5f95f7b622e4171096d92175dda0a560f0955ade9b8a3a07bdcf151f7359611 \ + --hash=sha256:e9acb2c4d9cb532c3fedea74159f7b923c8c036328c9239b4049e7aa073bdd81 \ + --hash=sha256:e9f924aa610c0618445e1e8738c822c3190ce2a2699a0cb48ec3a351a96761f2 \ + --hash=sha256:ed1a5891e59472884a03cb9875483e8fc131c80a275c60967f8afc5458a0c8ff \ + --hash=sha256:ed68e27b8a61e57a3ccdc7c5a14499e00b54dfe223087204d5d40b3b5ef58b6d \ + --hash=sha256:eeab73050ea58c13dd56e329f594c1dfe32ebd7bb169bbdf4f8ceefbc31ec6b5 \ + --hash=sha256:f4dafd6d6ababfa3b14dd6e5f0378cb7c7d291895a31a40abcbb7cc74f396131 \ + --hash=sha256:f69ec5be85ef508e206153bed8eafd03f7995dc464356c8bbb279a1e2b7d56f3 \ + --hash=sha256:f76d1562643693b8a40066f1f96af795b93fd9bcfc9690a1af2ff4c5867ee29e \ + --hash=sha256:f839d29d0cc12048cf073d88ca4fdf94d420bc2b8afd69641ff6d496422ccd4f \ + --hash=sha256:f9180c362bde06fd05380298ded4e234fbc0d6ede0a864835bfd91c1e24283d5 \ + --hash=sha256:f9ff356e97e3ab09db07c8b675efa67340103874a0bae7465acb83dad7a35f7f \ + --hash=sha256:fa74636a49fc8077413ce8db3e85f1c4aff880788bb55bda56253118e036fe5b + # via contextual-orchestrator (pyproject.toml) idna==3.18 \ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 @@ -403,6 +486,10 @@ pydantic-core==2.46.4 \ --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \ --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae # via pydantic +sortedcontainers==2.4.0 \ + --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \ + --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 + # via hypothesis sqlalchemy==2.0.51 \ --hash=sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23 \ --hash=sha256:0592bdadf86ddcabfd72d9ab66ea8a5d8d2cc6be1cc51fa7e66c03868ac5eac1 \ diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index be359ae59..5af76d2ec 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -145,6 +145,51 @@ def test_http_create_and_delete_worker_agents() -> None: assert {a.id for a in orchestrator.agents} == {"general_agent"} +def test_http_agent_registration_cannot_target_loopback_services() -> None: + """Keep local providers on the trusted startup boundary, not the HTTP API.""" + orchestrator = TaskOrchestrator(_seed()) + server = build_server( + orchestrator, + port=0, + security=SecurityConfig( + admin_token="admin_secret", + inference_token="inference_secret", + ), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + endpoint = ( + f"http://127.0.0.1:{server.server_address[1]}" + "/api/v1/agent_pools/default/worker_agents" + ) + local_agent = { + "id": "loopback_worker", + "model": "local-model", + "base_url": "mlx://127.0.0.1:6379/v1", + } + try: + inference_status, _ = _call( + endpoint, + "POST", + "inference_secret", + local_agent, + ) + admin_status, admin_body = _call( + endpoint, + "POST", + "admin_secret", + local_agent, + ) + finally: + server.shutdown() + thread.join(timeout=5) + + assert inference_status == 401 + assert admin_status == 400 + assert admin_body["error"]["code"] == "local_provider_registration_forbidden" + assert {agent.id for agent in orchestrator.candidates} == {"general_agent"} + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_") and callable(fn): diff --git a/tests/test_batch_api.py b/tests/test_batch_api.py index 57187ce04..abe340316 100644 --- a/tests/test_batch_api.py +++ b/tests/test_batch_api.py @@ -119,6 +119,37 @@ def test_batch_run_full_flow_over_http() -> None: assert b'name="purpose"' in provider.uploaded_jsonl +def test_batch_run_omits_an_unset_temperature() -> None: + """Do not invent a sampling field in the uploaded provider JSONL.""" + with _FakeBatchProvider() as provider: + agent = ModelAgent( + "worker_agent", + "reasoning-model", + base_url=provider.base_url, + api_key_env="UNSET_KEY_ENV", + ) + + _client()._batch_run( + agent, + REQUESTS, + temperature=None, + poll_interval=0.01, + poll_timeout=30, + ) + + assert b'"temperature"' not in provider.uploaded_jsonl + + _client()._batch_run( + agent, + REQUESTS, + temperature=0.2, + poll_interval=0.01, + poll_timeout=30, + ) + + assert b'"temperature": 0.2' in provider.uploaded_jsonl + + def test_batch_terminal_failure_raises() -> None: with _FakeBatchProvider(fail_status="failed") as provider: agent = ModelAgent("worker_agent", "gpt-x", base_url=provider.base_url, api_key_env="UNSET_KEY_ENV") diff --git a/tests/test_chat_temperature_top_p_http_honesty.py b/tests/test_chat_temperature_top_p_http_honesty.py index c080a7ed3..ded4046af 100644 --- a/tests/test_chat_temperature_top_p_http_honesty.py +++ b/tests/test_chat_temperature_top_p_http_honesty.py @@ -3,15 +3,17 @@ from __future__ import annotations import json +import sys import threading import urllib.error import urllib.request +from concurrent.futures import ThreadPoolExecutor from pathlib import Path -import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 _TEST_AUTH_TOKEN = "chat_temperature_top_p_http_honesty_token" # noqa: S105 @@ -200,6 +202,64 @@ def test_http_chat_accepts_sampling_omitted() -> None: thread.join(timeout=5) +def test_concurrent_http_sampling_is_request_scoped() -> None: + """An explicit temperature must not leak into a concurrent omitted request.""" + barrier = threading.Barrier(2) + observed: dict[str, float | None] = {} + + class ConcurrentClient(ModelClient): + """Hold two provider calls together so their HTTP request scopes overlap.""" + + def chat( + self, + agent: ModelAgent, + messages: list[dict], + temperature: float | None = None, + top_p: float | None = None, + ) -> str: + barrier.wait(timeout=5) + prompt = str(messages[-1]["content"]) + observed[prompt] = self._effective_temperature(temperature) + return f"[{agent.id}] answer" + + client = ConcurrentClient() + orchestrator = TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))], + client=client, + ) + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + explicit = { + "model": "mock-planner", + "messages": [{"role": "user", "content": "explicit"}], + "temperature": 0.2, + } + omitted = { + "model": "mock-planner", + "messages": [{"role": "user", "content": "omitted"}], + } + with ThreadPoolExecutor(max_workers=2) as pool: + results = list( + pool.map( + lambda payload: _post(port, "/v1/chat/completions", payload), + (explicit, omitted), + ) + ) + assert all(status == 200 for status, _body in results), results + assert observed == {"explicit": 0.2, "omitted": None} + assert client.default_temperature is None + finally: + server.shutdown() + thread.join(timeout=5) + + if __name__ == "__main__": test_http_chat_accepts_temperature_and_top_p_in_range() test_http_chat_rejects_temperature_above_two() @@ -209,4 +269,5 @@ def test_http_chat_accepts_sampling_omitted() -> None: test_http_chat_rejects_top_p_above_one() test_http_completions_accepts_temperature_bounds() test_http_chat_accepts_sampling_omitted() + test_concurrent_http_sampling_is_request_scoped() print("ok") diff --git a/tests/test_kv_credentials.py b/tests/test_kv_credentials.py index 52e41ccd7..4735d7e89 100644 --- a/tests/test_kv_credentials.py +++ b/tests/test_kv_credentials.py @@ -6,8 +6,9 @@ from __future__ import annotations import os -from pathlib import Path +import socket import sys +from pathlib import Path import pytest @@ -147,8 +148,16 @@ def open_provider(request, _destination=None): seen.append(request) return _Response() - with patch.object(client, "_open_provider", side_effect=open_provider): + with ( + patch.object( + client, + "_resolve_addresses", + return_value=[(socket.AF_INET, ("93.184.216.34", 443))], + ) as resolve_addresses, + patch.object(client, "_open_provider", side_effect=open_provider), + ): assert client.chat(agent, [{"role": "user", "content": "ping"}]) == "ok" + resolve_addresses.assert_called_once_with("api.bytez.com", 443) assert seen[0].get_header("Authorization") == "Key bytez-secret" diff --git a/tests/test_local_mlx.py b/tests/test_local_mlx.py index 8c2b3d815..c419fe8b4 100644 --- a/tests/test_local_mlx.py +++ b/tests/test_local_mlx.py @@ -180,7 +180,7 @@ def open_provider(request, _destination=None, *, timeout=None): payload = json.loads(seen[1][0].data) assert payload["max_tokens"] == 1 - assert payload["temperature"] == 0.0 + assert "temperature" not in payload assert payload["chat_template_kwargs"] == {"enable_thinking": False} diff --git a/tests/test_model_judge.py b/tests/test_model_judge.py index 56381a9f5..b19d54772 100644 --- a/tests/test_model_judge.py +++ b/tests/test_model_judge.py @@ -306,7 +306,6 @@ def test_fast_mlsirm_adapter_routes_structured_completion_through_gateway() -> N { "model": "model-x", "messages": [{"role": "user", "content": "judge"}], - "temperature": orchestrator.client.temperature, "max_tokens": orchestrator.client.max_output_tokens, "response_format": response_format, } @@ -316,6 +315,29 @@ def test_fast_mlsirm_adapter_routes_structured_completion_through_gateway() -> N assert completion["trace"][0]["usage"]["total_tokens"] == 5 +def test_fast_mlsirm_structured_completion_forwards_request_scoped_temperature() -> None: + orchestrator, _ = _orch("unused") + adapter = orchestrator_module._FastMLSIJudgeAdapter( + orchestrator, + "task", + "general_agent", + mode="route", + ) + orchestrator.client.default_temperature = 0.3 + response_format = {"type": "json_object"} + with patch.object( + orchestrator, + "proxy_completion", + return_value={"choices": [{"message": {"content": "{}"}}]}, + ) as proxy: + adapter.complete_structured( + [{"role": "user", "content": "judge"}], + response_format=response_format, + ) + + assert proxy.call_args.args[0]["temperature"] == 0.3 + + def test_fast_mlsirm_judge_contract_does_not_pass_threshold_to_judge_call() -> None: class _Judge: def __init__(self, _orchestrator, *, mode: str, accept_threshold: float) -> None: diff --git a/tests/test_sampling_defaults.py b/tests/test_sampling_defaults.py new file mode 100644 index 000000000..eedbe7994 --- /dev/null +++ b/tests/test_sampling_defaults.py @@ -0,0 +1,204 @@ +"""Sampling defaults must not disable providers with narrower capabilities.""" + +from __future__ import annotations + +from unittest.mock import patch + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.orchestrator import ModelClient, _FastMLSIJudgeAdapter + + +def _remote_agent() -> ModelAgent: + return ModelAgent( + "reasoning_agent", + "reasoning-model", + base_url="https://provider.example/v1", + credential_key="TEST_PROVIDER_API_KEY", + ) + + +def test_chat_omits_temperature_when_caller_does_not_provide_one() -> None: + client = ModelClient() + captured: list[dict] = [] + + def send(_agent, payload, _destination): + captured.append(payload) + return "answer" + + with ( + patch.object(client, "_validate_provider", return_value=object()), + patch("contextual_orchestrator.orchestrator._provider_credential", return_value="secret"), + patch.object(client, "_send_with_retry", side_effect=send), + ): + assert client.chat(_remote_agent(), [{"role": "user", "content": "hello"}]) == "answer" + + assert "temperature" not in captured[0] + + +def test_chat_forwards_an_explicit_temperature() -> None: + client = ModelClient() + captured: list[dict] = [] + + def send(_agent, payload, _destination): + captured.append(payload) + return "answer" + + with ( + patch.object(client, "_validate_provider", return_value=object()), + patch("contextual_orchestrator.orchestrator._provider_credential", return_value="secret"), + patch.object(client, "_send_with_retry", side_effect=send), + ): + client.chat( + _remote_agent(), + [{"role": "user", "content": "hello"}], + temperature=0.2, + ) + + assert captured[0]["temperature"] == 0.2 + + +def test_chat_uses_request_scoped_temperature_before_constructor_value() -> None: + client = ModelClient(temperature=0.4) + captured: list[dict] = [] + + def send(_agent, payload, _destination): + captured.append(payload) + return "answer" + + with ( + patch.object(client, "_validate_provider", return_value=object()), + patch("contextual_orchestrator.orchestrator._provider_credential", return_value="secret"), + patch.object(client, "_send_with_retry", side_effect=send), + ): + client.chat( + _remote_agent(), + [{"role": "user", "content": "hello"}], + temperature=0.3, + ) + + assert captured[0]["temperature"] == 0.3 + + +def test_chat_uses_constructor_temperature_when_request_scope_is_unset() -> None: + client = ModelClient(temperature=0.4) + captured: list[dict] = [] + + def send(_agent, payload, _destination): + captured.append(payload) + return "answer" + + with ( + patch.object(client, "_validate_provider", return_value=object()), + patch("contextual_orchestrator.orchestrator._provider_credential", return_value="secret"), + patch.object(client, "_send_with_retry", side_effect=send), + ): + client.chat(_remote_agent(), [{"role": "user", "content": "hello"}]) + + assert captured[0]["temperature"] == 0.4 + + +def test_stream_chat_uses_request_temperature_before_constructor_value() -> None: + """Forward a request-scoped temperature on streamed route completions.""" + client = ModelClient(temperature=0.4) + captured: list[dict] = [] + + def stream_send(_agent, payload, _destination): + captured.append(payload) + yield "answer" + + with ( + patch.object(client, "_validate_provider", return_value=object()), + patch.object(client, "_stream_send", side_effect=stream_send), + ): + with client.request_options( + max_output_tokens=64, + top_p=0.9, + presence_penalty=0.1, + frequency_penalty=0.2, + ): + assert list( + client.stream_chat( + _remote_agent(), + [{"role": "user", "content": "hello"}], + temperature=0.3, + ) + ) == ["answer"] + + assert captured[0]["temperature"] == 0.3 + assert captured[0]["max_tokens"] == 64 + assert captured[0]["top_p"] == 0.9 + assert captured[0]["presence_penalty"] == 0.1 + assert captured[0]["frequency_penalty"] == 0.2 + + +def test_nested_request_options_restore_thread_local_values() -> None: + """Nested request scopes restore the outer value and then the client default.""" + client = ModelClient() + + with client.request_options(temperature=0.4): + assert client._effective_temperature() == 0.4 + with client.request_options(max_output_tokens=73, temperature=0.2): + assert client._effective_max_output_tokens() == 73 + assert client._effective_temperature() == 0.2 + assert client._effective_temperature() == 0.4 + + assert client._effective_max_output_tokens() == client.max_output_tokens + assert client._effective_temperature() is None + + +def test_structured_completion_omits_an_unset_temperature() -> None: + """Do not turn an omitted structured sampling value into JSON null.""" + orchestrator = TaskOrchestrator([_remote_agent()], client=ModelClient()) + adapter = _FastMLSIJudgeAdapter( + orchestrator=orchestrator, + text="judge this answer", + judge="reasoning_agent", + ) + captured: list[dict] = [] + + def proxy(payload): + captured.append(payload) + return { + "choices": [{"message": {"content": '{"decision":"ACCEPT","reason":"ok"}'}}] + } + + with ( + orchestrator.client.request_options(max_output_tokens=73), + patch.object(orchestrator, "proxy_completion", side_effect=proxy), + ): + result = adapter.complete_structured( + [{"role": "user", "content": "judge"}], + response_format={"type": "json_object"}, + ) + + assert result["answer"].startswith("{") + assert captured[0]["max_tokens"] == 73 + assert "temperature" not in captured[0] + + +def test_structured_completion_forwards_an_explicit_client_temperature() -> None: + """Preserve an explicitly configured sampling value for structured output.""" + orchestrator = TaskOrchestrator( + [_remote_agent()], + client=ModelClient(temperature=0.3), + ) + adapter = _FastMLSIJudgeAdapter( + orchestrator=orchestrator, + text="judge this answer", + judge="reasoning_agent", + ) + captured: list[dict] = [] + + def proxy(payload): + captured.append(payload) + return { + "choices": [{"message": {"content": '{"decision":"ACCEPT","reason":"ok"}'}}] + } + + with patch.object(orchestrator, "proxy_completion", side_effect=proxy): + adapter.complete_structured( + [{"role": "user", "content": "judge"}], + response_format={"type": "json_object"}, + ) + + assert captured[0]["temperature"] == 0.3