diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index d3943c5be..5322faa7c 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -222,6 +222,13 @@ class UsageRecord: def as_dict(self) -> Dict[str, Any]: """Flatten the record (attribution inlined) for JSON + SQL storage.""" + # Prefer an explicit attribution model_name (client tag) for rollups; + # otherwise the served model id on the record. + rollup_model = ( + self.attribution.model_name + if self.attribution.model_name != UNATTRIBUTED + else self.model_name + ) row = { "usage_record_id": self.usage_record_id, "created_at": self.created_at, @@ -229,7 +236,7 @@ def as_dict(self) -> Dict[str, Any]: "request_channel": self.request_channel, "route_mode": self.route_mode, "provider_name": self.provider_name, - "model_name": self.model_name, + "model_name": rollup_model, "prompt_tokens": self.prompt_tokens, "completion_tokens": self.completion_tokens, "total_tokens": self.total_tokens, @@ -583,12 +590,12 @@ def _seed_dimension_catalog(self) -> None: ph = self._placeholder() cur = self._conn.cursor() for order, (name, label, _column) in enumerate(ATTRIBUTION_DIMENSION_CATALOG): - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: only the DB-API placeholder char is interpolated; the value is bound. f"SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = {ph}", # nosec B608 - ph is a DB-API placeholder. (name,), ) if cur.fetchone() is None: - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: only DB-API placeholder chars are interpolated; values are bound. "INSERT INTO cost_attribution_dimensions " f"(dimension_name, dimension_label, dimension_order) VALUES ({ph}, {ph}, {ph})", # nosec B608 - ph is a DB-API placeholder. (name, label, order), @@ -602,7 +609,7 @@ def append(self, record: UsageRecord) -> None: placeholders = ", ".join(ph for _ in _USAGE_COLUMNS) columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: columns are the fixed _USAGE_COLUMNS constant; values are bound. f"INSERT INTO llm_usage_records ({columns}) VALUES ({placeholders})", # nosec B608 - columns are fixed _USAGE_COLUMNS. tuple(row.get(column) for column in _USAGE_COLUMNS), ) @@ -622,7 +629,7 @@ def query(self, start: Optional[int] = None, end: Optional[int] = None) -> List[ where = f" WHERE {' AND '.join(clauses)}" if clauses else "" columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. + cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. # nosemgrep -- sqlalchemy-execute-raw-query FP: fixed columns and clause templates; all values are bound. return [dict(zip(_USAGE_COLUMNS, values)) for values in cur.fetchall()] diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0097b722e..22878d765 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -215,6 +215,10 @@ def __init__( ) -> None: self.timeout = timeout self.max_output_tokens = max_output_tokens + self.default_temperature = 0.2 + self.default_top_p: float | None = None + self.default_presence_penalty: float | None = None + self.default_frequency_penalty: float | None = None self.max_retries = max_retries self.retry_backoff = retry_backoff self.retry_backoff_cap = retry_backoff_cap @@ -230,7 +234,7 @@ def __init__( @staticmethod def _build_ssl_context(ca_bundle: str | None, verify_tls: bool) -> ssl.SSLContext: if not verify_tls: - return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. + return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. # nosemgrep -- unverified-ssl-context: intentional, default-secure (verify_tls defaults True) dev-only opt-out for self-signed endpoints. if ca_bundle: if not os.path.isfile(ca_bundle): raise ValueError(f"provider CA bundle does not exist: {ca_bundle}") @@ -246,9 +250,29 @@ def take_usage(self) -> dict[str, Any] | None: self._local.usage = None return usage - def chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: float = 0.2) -> str: - """Send messages to a mock or OpenAI-compatible chat endpoint with retries.""" + def chat( + self, + agent: ModelAgent, + messages: list[ChatMessage], + temperature: float | None = None, + top_p: float | None = None, + ) -> str: + """Send messages to a mock or OpenAI-compatible chat endpoint with retries. + + When ``temperature``/``top_p`` are omitted, ``default_temperature`` and + ``default_top_p`` are used so request-scoped Completions sampling can be + applied without threading kwargs through every orchestrator hop. + """ self._local.usage = None + # Expose the effective sampling knobs for request-path tests / diagnostics. + effective_temperature = self.default_temperature if temperature is None else temperature + effective_top_p = self.default_top_p if top_p is None else top_p + effective_presence = self.default_presence_penalty + effective_frequency = self.default_frequency_penalty + self._local.last_temperature = effective_temperature + self._local.last_top_p = effective_top_p + self._local.last_presence_penalty = effective_presence + self._local.last_frequency_penalty = effective_frequency if agent.base_url.startswith("mock://"): return self._mock(agent, messages) @@ -262,10 +286,16 @@ def chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: floa payload = { # pragma: no cover "model": agent.model, "messages": messages, - "temperature": temperature, + "temperature": effective_temperature, "stream": False, "max_tokens": self.max_output_tokens, } + if effective_top_p is not None: # pragma: no cover + payload["top_p"] = effective_top_p + if effective_presence is not None: # pragma: no cover + payload["presence_penalty"] = effective_presence + if effective_frequency is not None: # pragma: no cover + payload["frequency_penalty"] = effective_frequency return self._send_with_retry(agent, payload) def _send_with_retry(self, agent: ModelAgent, payload: dict[str, Any]) -> str: @@ -307,7 +337,7 @@ def _send(self, agent: ModelAgent, payload: dict[str, Any]) -> str: def _open_provider(self, request: urllib.request.Request) -> Any: """Open a provider request built from a validated provider URL.""" - return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. + return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. # nosemgrep -- dynamic-urllib-use: URL is built by _provider_url after scheme/host validation; egress to loopback/private/reserved is blocked. request, timeout=self.timeout, context=self._ssl_context, @@ -897,7 +927,23 @@ def proxy_completion( text = self._latest_user_text(messages) else: text = _coerce_input_text(body.get("input")) - agent = self._select_agent(text, "worker") + requested_model = body.get("model") + # When the client names a model, resolve a pool agent that actually serves + # that model id. Silent rewrite to an unrelated agent.model is a commercial + # honesty failure for OpenAI SDKs (passthrough tools/Responses paths). + if isinstance(requested_model, str) and requested_model.strip(): + matched = [ + agent + for agent in self.agents + if not getattr(agent, "disabled", False) and agent.model == requested_model + ] + if not matched: + raise ValueError( + f"model {requested_model!r} is not available in the agent pool" + ) + agent = matched[0] + else: + agent = self._select_agent(text, "worker") upstream = { key: value for key, value in body.items() @@ -8517,6 +8563,29 @@ def chat_completion_response( } +def text_completion_response( + result: dict[str, Any], + model: str = "contextual-orchestrator", + usage: dict[str, int] | None = None, +) -> dict[str, Any]: # pragma: no cover + """Wrap orchestration output as OpenAI legacy ``text_completion`` (``/v1/completions``).""" + return { + "id": f"cmpl-{int(time.time() * 1000)}", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "text": result["answer"], + "logprobs": None, + "finish_reason": "stop", + } + ], + "usage": usage or {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + _STREAM_CHUNK_SIZE = 32 diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index c58d4cb79..e17bec7a3 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -22,6 +22,7 @@ TaskOrchestrator, chat_completion_chunks, chat_completion_response, + text_completion_response, redact_value, sse_stream_body, ) @@ -32,7 +33,7 @@ "seed", "presence_penalty", "frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "user", "metadata", "parallel_tool_calls", "reasoning_effort", "response_format", "tools", "tool_choice", "functions", "function_call", - "modalities", "prediction", "store", "service_tier", + "modalities", "prediction", "store", "service_tier", "stream_options", } # Provider features the multi-agent verifier cannot merge -> single-agent passthrough. PASSTHROUGH_TRIGGER_KEYS = {"response_format", "tools", "tool_choice", "functions", "function_call"} @@ -46,6 +47,14 @@ } | OPENAI_PASSTHROUGH_PARAM_KEYS ALLOWED_BATCH_KEYS = {"requests", "attribution", "routing", "model"} ALLOWED_EMBEDDINGS_BATCH_KEYS = {"model", "input", "inputs", "endpoint", "metadata", "attribution"} +ALLOWED_EMBEDDINGS_KEYS = { + "model", "input", "encoding_format", "dimensions", "user", "metadata", "attribution", +} +ALLOWED_COMPLETIONS_KEYS = { + "model", "prompt", "stream", "stream_options", "echo", "suffix", "best_of", + "logprobs", "n", "max_tokens", "temperature", "top_p", "stop", "user", "seed", + "presence_penalty", "frequency_penalty", "logit_bias", "service_tier", "metadata", +} | {"attribution", "routing"} ALLOWED_MESSAGE_ROLES = {"system", "user", "assistant", "tool"} ALLOWED_MODES = {"auto", "route", "conduct"} ALLOWED_SIMULATE_KEYS = {"prompt", "mode", "include_orchestration_trace"} @@ -171,6 +180,554 @@ def _coerce_json(payload: bytes) -> dict[str, Any]: return value + +def _validate_completion_prompt(prompt: Any) -> list[dict[str, str]]: + """Legacy Completions ``prompt`` → single user message list. + + Accepts a non-empty string or an array of strings (at most 128 items). OpenAI + also allows arrays of token IDs (integers); this gateway rejects token-id + prompts fail-closed with ``invalid_prompt`` so SDKs get a clear migration + path to string prompts. + """ + if isinstance(prompt, str): + if not prompt.strip(): + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + if len(prompt) > 32_000: + raise RequestError(400, "invalid_prompt", "prompt must be at most 32000 characters") + return [{"role": "user", "content": prompt}] + if isinstance(prompt, list): + if not prompt: + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + if len(prompt) > 128: + raise RequestError( + 400, + "invalid_prompt", + "prompt array must contain at most 128 items", + ) + # Token-id form: list of ints, or list of list of ints (batch of token sequences). + if all(isinstance(item, int) and not isinstance(item, bool) for item in prompt): + raise RequestError( + 400, + "invalid_prompt", + "token-id prompts are not supported; pass a string or array of strings", + ) + if all(isinstance(item, list) for item in prompt): + raise RequestError( + 400, + "invalid_prompt", + "token-id prompts are not supported; pass a string or array of strings", + ) + parts: list[str] = [] + for item in prompt: + if not isinstance(item, str): + raise RequestError(400, "invalid_prompt", "prompt array items must be strings") + if not item.strip(): + raise RequestError( + 400, + "invalid_prompt", + "prompt array items must be non-empty strings", + ) + parts.append(item) + joined = "\n".join(parts) + if not joined.strip(): + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + if len(joined) > 32_000: + raise RequestError(400, "invalid_prompt", "prompt must be at most 32000 characters") + return [{"role": "user", "content": joined}] + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + + +def _validate_completions_stream(body: dict[str, Any]) -> bool | None: + """Legacy Completions ``stream`` — strict boolean honesty contract. + + OpenAI Completions accepts streaming. This gateway: + - accepts omit and ``stream=false`` as the non-streaming text_completion path + - rejects ``stream=true`` with a clear redirect to chat completions + - rejects non-boolean values fail-closed (no silent coercion) + """ + if "stream" not in body: + return None + stream = body.get("stream") + if not isinstance(stream, bool): + raise RequestError(400, "invalid_stream", "stream must be a boolean") + if stream is True: + raise RequestError( + 400, + "invalid_stream", + "stream is not supported on /v1/completions; use /v1/chat/completions", + ) + return stream + + +def _validate_completions_echo(body: dict[str, Any]) -> bool | None: + """Legacy Completions ``echo`` — strict boolean; ``true`` is not supported. + + OpenAI can prepend the prompt to the completion when ``echo`` is true. This + gateway does not implement that behaviour, so ``echo=true`` fails closed with + a clear ``invalid_echo`` error. ``false`` and omit remain valid. + """ + if "echo" not in body: + return None + echo = body.get("echo") + if not isinstance(echo, bool): + raise RequestError(400, "invalid_echo", "echo must be a boolean") + if echo is True: + raise RequestError( + 400, + "invalid_echo", + "echo=true is not supported on /v1/completions", + ) + return echo + + + + + + + + + + + + + + +def _validate_completions_logit_bias(body: dict[str, Any]) -> dict[str, float] | None: + """Legacy Completions ``logit_bias`` — empty object is a no-op; non-empty fails closed. + + OpenAI uses logit_bias to bias token sampling. This gateway does not apply + token biases on the Completions route. An empty object is an honest no-op + (SDK clients often send ``{}``). Any non-empty map is type-checked then + rejected so clients never believe sampling bias was applied. + """ + if "logit_bias" not in body: + return None + bias = body.get("logit_bias") + if not isinstance(bias, dict): + raise RequestError(400, "invalid_logit_bias", "logit_bias must be an object of token biases") + # Empty object: no tokens to bias — treat as omit (honest no-op). + if len(bias) == 0: + return {} + if len(bias) > 300: + raise RequestError(400, "invalid_logit_bias", "logit_bias must contain at most 300 entries") + for key, value in bias.items(): + token = str(key) + if not token.isdigit(): + raise RequestError(400, "invalid_logit_bias", "logit_bias keys must be digit token ids") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RequestError(400, "invalid_logit_bias", "logit_bias values must be numbers in [-100, 100]") + number = float(value) + if number < -100 or number > 100: + raise RequestError(400, "invalid_logit_bias", "logit_bias values must be numbers in [-100, 100]") + raise RequestError( + 400, + "invalid_logit_bias", + "logit_bias is not supported on /v1/completions", + ) + + + +def _validate_service_tier(body: dict[str, Any], *, endpoint_path: str) -> str | None: + """OpenAI ``service_tier`` — accept omit/auto/default as no-ops; reject others. + + OpenAI uses service_tier for capacity priority (auto/default/flex/priority). + This gateway has no tiered capacity plane, so only auto/default (or omit) + are honest no-ops. Other values fail closed so clients cannot silently + believe flex/priority processing was applied. + """ + if "service_tier" not in body: + return None + service_tier = body.get("service_tier") + if not isinstance(service_tier, str): + raise RequestError(400, "invalid_service_tier", "service_tier must be a string") + if service_tier not in ("auto", "default"): + raise RequestError( + 400, + "invalid_service_tier", + f"service_tier values other than auto or default are not supported on {endpoint_path}", + ) + return service_tier + + +def _validate_completions_user(body: dict[str, Any]) -> str | None: + """Legacy Completions ``user`` — optional string end-user id, max 64 characters.""" + if "user" not in body: + return None + user = body.get("user") + if not isinstance(user, str): + raise RequestError(400, "invalid_user", "user must be a string of at most 64 characters") + if not user.strip(): + raise RequestError(400, "invalid_user", "user must be a non-empty string of at most 64 characters") + if len(user) > 64: + raise RequestError(400, "invalid_user", "user must be a string of at most 64 characters") + return user + +def _validate_completions_n(body: dict[str, Any]) -> int | None: + """Legacy Completions ``n`` — positive integer; only ``n=1`` is supported. + + OpenAI can return multiple completions when ``n > 1``. This gateway always + returns a single choice, so ``n > 1`` fails closed. ``n=1`` and omit remain + valid. Cap 128 is retained for clear range errors before the support check. + """ + if "n" not in body: + return None + n = body.get("n") + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise RequestError(400, "invalid_n", "n must be a positive integer") + if n > 128: + raise RequestError(400, "invalid_n", "n must be at most 128") + if n > 1: + raise RequestError( + 400, + "invalid_n", + "n greater than 1 is not supported on /v1/completions", + ) + return n + +def _validate_completions_stop(body: dict[str, Any]) -> str | list[str] | None: + """Legacy Completions ``stop`` — type-checked then rejected (not applied). + + OpenAI uses stop sequences to cut generation early. This gateway validates + shape (string or ≤4 non-empty strings, each ≤256 chars) but does not apply + stop sequences on the Completions path, so any provided ``stop`` fails closed. + """ + if "stop" not in body: + return None + stop = body.get("stop") + if isinstance(stop, str): + if not stop: + raise RequestError(400, "invalid_stop", "stop sequences must be non-empty strings") + if len(stop) > 256: + raise RequestError(400, "invalid_stop", "each stop sequence must be at most 256 characters") + elif isinstance(stop, list): + if not stop or len(stop) > 4: + raise RequestError(400, "invalid_stop", "stop must be a string or array of up to 4 non-empty strings") + for item in stop: + if not isinstance(item, str) or not item: + raise RequestError(400, "invalid_stop", "stop sequences must be non-empty strings") + if len(item) > 256: + raise RequestError(400, "invalid_stop", "each stop sequence must be at most 256 characters") + else: + raise RequestError(400, "invalid_stop", "stop must be a string or array of up to 4 non-empty strings") + raise RequestError( + 400, + "invalid_stop", + "stop sequences are not supported on /v1/completions", + ) + + + +def _validate_completions_seed(body: dict[str, Any]) -> int | None: + """Legacy Completions ``seed`` — type-checked then rejected (not applied). + + OpenAI uses seed for best-effort deterministic sampling. This gateway validates + signed int64 integers but does not apply seed on the Completions route path, + so any provided ``seed`` fails closed. Omit remains valid. + """ + if "seed" not in body: + return None + seed = body.get("seed") + if isinstance(seed, bool) or not isinstance(seed, int): + raise RequestError(400, "invalid_seed", "seed must be an integer") + if seed < -(2**63) or seed > (2**63 - 1): + raise RequestError(400, "invalid_seed", "seed must fit in a signed 64-bit integer") + raise RequestError( + 400, + "invalid_seed", + "seed is not supported on /v1/completions", + ) + + + +def _validate_completions_frequency_penalty(body: dict[str, Any]) -> float | None: + """Legacy Completions ``frequency_penalty`` — number in [-2, 2].""" + if "frequency_penalty" not in body: + return None + value = body.get("frequency_penalty") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RequestError(400, "invalid_frequency_penalty", "frequency_penalty must be a number in [-2, 2]") + number = float(value) + if number < -2 or number > 2: + raise RequestError(400, "invalid_frequency_penalty", "frequency_penalty must be a number in [-2, 2]") + return number + +def _validate_completions_presence_penalty(body: dict[str, Any]) -> float | None: + """Legacy Completions ``presence_penalty`` — number in [-2, 2].""" + if "presence_penalty" not in body: + return None + value = body.get("presence_penalty") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RequestError(400, "invalid_presence_penalty", "presence_penalty must be a number in [-2, 2]") + number = float(value) + if number < -2 or number > 2: + raise RequestError(400, "invalid_presence_penalty", "presence_penalty must be a number in [-2, 2]") + return number + +def _validate_completions_temperature(body: dict[str, Any]) -> float | None: + """Legacy Completions ``temperature`` — number in [0, 2].""" + if "temperature" not in body: + return None + temperature = body.get("temperature") + if isinstance(temperature, bool) or not isinstance(temperature, (int, float)): + raise RequestError(400, "invalid_temperature", "temperature must be a number in [0, 2]") + value = float(temperature) + if value < 0 or value > 2: + raise RequestError(400, "invalid_temperature", "temperature must be a number in [0, 2]") + return value + +def _validate_completions_top_p(body: dict[str, Any]) -> float | None: + """Legacy Completions ``top_p`` — number in (0, 1] (OpenAI nucleus sampling).""" + if "top_p" not in body: + return None + top_p = body.get("top_p") + if isinstance(top_p, bool) or not isinstance(top_p, (int, float)): + raise RequestError(400, "invalid_top_p", "top_p must be a number in (0, 1]") + value = float(top_p) + if value <= 0 or value > 1: + raise RequestError(400, "invalid_top_p", "top_p must be a number in (0, 1]") + return value + +def _validate_completions_model(body: dict[str, Any]) -> str: + """Legacy Completions ``model`` — required non-empty string (OpenAI parity).""" + if "model" not in body: + raise RequestError(400, "invalid_model", "model is required") + model = body.get("model") + if not isinstance(model, str) or not model.strip(): + raise RequestError(400, "invalid_model", "model must be a non-empty string") + if len(model) > 256: + raise RequestError(400, "invalid_model", "model must be at most 256 characters") + return model + +def _validate_completions_max_tokens(body: dict[str, Any]) -> int | None: + """Legacy Completions ``max_tokens`` — positive integer capped at 1_048_576.""" + if "max_tokens" not in body: + return None + max_tokens = body.get("max_tokens") + if isinstance(max_tokens, bool) or not isinstance(max_tokens, int) or max_tokens < 1: + raise RequestError(400, "invalid_max_tokens", "max_tokens must be a positive integer") + if max_tokens > 1_048_576: + raise RequestError( + 400, + "invalid_max_tokens", + "max_tokens must be at most 1048576", + ) + return max_tokens + +def _validate_chat_max_completion_tokens(body: dict[str, Any]) -> int | None: + """Chat Completions ``max_completion_tokens`` — positive integer capped at 1_048_576. + + OpenAI prefers this over legacy ``max_tokens`` for chat. When both are set, + ``max_completion_tokens`` wins so clients get a single honest budget. + """ + if "max_completion_tokens" not in body: + return None + max_completion_tokens = body.get("max_completion_tokens") + if ( + isinstance(max_completion_tokens, bool) + or not isinstance(max_completion_tokens, int) + or max_completion_tokens < 1 + ): + raise RequestError( + 400, + "invalid_max_completion_tokens", + "max_completion_tokens must be a positive integer", + ) + if max_completion_tokens > 1_048_576: + raise RequestError( + 400, + "invalid_max_completion_tokens", + "max_completion_tokens must be at most 1048576", + ) + return max_completion_tokens + + +def _validate_completions_logprobs(body: dict[str, Any]) -> int | bool | None: + """Legacy Completions ``logprobs`` — only ``false``/omit; token logprobs unsupported. + + OpenAI accepts ``false`` or an integer 0–5 for top logprob counts. This gateway + always returns ``logprobs: null`` on text completions, so integer logprobs + (including 0–5) and boolean ``true`` fail closed. ``false`` and omit remain valid. + """ + if "logprobs" not in body: + return None + logprobs = body.get("logprobs") + if logprobs is False: + return False + if isinstance(logprobs, bool): # True + raise RequestError( + 400, + "invalid_logprobs", + "logprobs must be false; token logprobs are not supported on /v1/completions", + ) + if isinstance(logprobs, int) and not isinstance(logprobs, bool): + raise RequestError( + 400, + "invalid_logprobs", + "token logprobs are not supported on /v1/completions; pass false or omit", + ) + raise RequestError( + 400, + "invalid_logprobs", + "logprobs must be false; token logprobs are not supported on /v1/completions", + ) + +def _validate_completions_suffix(body: dict[str, Any]) -> str | None: + """Legacy Completions ``suffix`` — optional string; non-empty is not supported. + + OpenAI appends ``suffix`` after the model completion. This gateway does not + implement that insertion, so a non-empty suffix fails closed. Empty string + and omit remain valid. Non-string values and oversized strings still fail. + """ + if "suffix" not in body: + return None + suffix = body.get("suffix") + if not isinstance(suffix, str): + raise RequestError(400, "invalid_suffix", "suffix must be a string") + if len(suffix) > 8_000: + raise RequestError(400, "invalid_suffix", "suffix must be at most 8000 characters") + if suffix: + raise RequestError( + 400, + "invalid_suffix", + "non-empty suffix is not supported on /v1/completions", + ) + return suffix + + +def _validate_completions_best_of(body: dict[str, Any]) -> int | None: + """Legacy Completions ``best_of`` — positive integer, ``best_of >= n``, max 1. + + OpenAI generates ``best_of`` candidates server-side and returns the top ``n``. + This gateway runs a single completion path, so ``best_of > 1`` fails closed + rather than silently returning one unranked candidate. ``best_of=1`` (and + omit) remain valid. Boolean ``True``/``False`` are rejected. + """ + if "best_of" not in body: + return None + best_of = body.get("best_of") + if isinstance(best_of, bool) or not isinstance(best_of, int) or best_of < 1: + raise RequestError(400, "invalid_best_of", "best_of must be a positive integer") + if best_of > 128: + raise RequestError(400, "invalid_best_of", "best_of must be at most 128") + if best_of > 1: + raise RequestError( + 400, + "invalid_best_of", + "best_of greater than 1 is not supported on /v1/completions", + ) + n = body.get("n", 1) + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise RequestError(400, "invalid_n", "n must be a positive integer") + if best_of < n: + raise RequestError( + 400, + "invalid_best_of", + "best_of must be greater than or equal to n", + ) + return best_of + + +def _validate_completions_stream_options(body: dict[str, Any]) -> dict[str, Any] | None: + """Legacy Completions ``stream_options`` — object with boolean flags; requires stream=true. + + Mirrors OpenAI chat Completions: ``stream_options`` is only valid when streaming. + This gateway rejects Completions streaming, so a well-formed ``stream_options`` + still fails closed once ``stream`` is checked (or here if ``stream`` is not true). + """ + if "stream_options" not in body: + return None + opts = body.get("stream_options") + if not isinstance(opts, dict): + raise RequestError(400, "invalid_stream_options", "stream_options must be an object") + if body.get("stream") is not True: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options requires stream=true", + ) + allowed = {"include_usage", "include_obfuscation"} + unknown = sorted(set(opts) - allowed) + if unknown: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options contains unsupported fields", + {"fields": unknown}, + ) + if "include_usage" in opts and not isinstance(opts["include_usage"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_usage must be a boolean", + ) + if "include_obfuscation" in opts and not isinstance(opts["include_obfuscation"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_obfuscation must be a boolean", + ) + return opts + + + + +def _validate_chat_stream_options(body: dict[str, Any], stream: bool) -> dict[str, Any] | None: + """Chat Completions ``stream_options`` — requires stream=true; include_usage unsupported. + + Shape matches OpenAI (include_usage / include_obfuscation booleans). This + gateway's SSE route path does not emit a final usage chunk and does not + apply stream obfuscation, so include_usage/include_obfuscation=true fail closed. + """ + if "stream_options" not in body: + return None + opts = body.get("stream_options") + if not isinstance(opts, dict): + raise RequestError(400, "invalid_stream_options", "stream_options must be an object") + if stream is not True: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options requires stream=true on /v1/chat/completions", + ) + allowed = {"include_usage", "include_obfuscation"} + unknown = sorted(set(opts) - allowed) + if unknown: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options contains unsupported fields", + {"fields": unknown}, + ) + if "include_usage" in opts: + if not isinstance(opts["include_usage"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_usage must be a boolean", + ) + if opts["include_usage"] is True: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_usage=true is not supported on /v1/chat/completions", + ) + if "include_obfuscation" in opts: + if not isinstance(opts["include_obfuscation"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_obfuscation must be a boolean", + ) + if opts["include_obfuscation"] is True: + # SSE obfuscation is not applied by this gateway; fail closed. + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_obfuscation=true is not supported on /v1/chat/completions", + ) + return opts + + def _reject_unknown_keys(body: dict[str, Any], allowed: set[str]) -> None: unknown = sorted(set(body) - allowed) if unknown: @@ -183,6 +740,26 @@ def _validate_mode(mode: Any) -> str: return mode + +def _require_pool_model(orchestrator: Any, model_name: str) -> None: + """Fail closed when ``model_name`` is not served by any enabled agent. + + OpenAI clients treat ``model`` as the deployment they paid for. Silently + answering with a different pool agent hides capacity/routing mismatches. + """ + agents = getattr(orchestrator, "agents", None) or [] + for agent in agents: + if getattr(agent, "disabled", False): + continue + if getattr(agent, "model", None) == model_name: + return + raise RequestError( + 400, + "invalid_model", + f"model {model_name!r} is not available in the agent pool", + ) + + def _validate_messages(messages: Any) -> list[dict[str, str]]: if not isinstance(messages, list) or not messages: raise RequestError(400, "invalid_message", "messages must be a non-empty array") @@ -192,9 +769,238 @@ def _validate_messages(messages: Any) -> list[dict[str, str]]: raise RequestError(400, "invalid_message", "each message must be an object") role = message.get("role") content = message.get("content") + if isinstance(role, str) and role == "developer": + # Newer OpenAI clients send developer in place of system; this gateway + # does not apply a separate developer plane — fail closed with migration. + raise RequestError( + 400, + "invalid_message_role", + "developer role is not supported on /v1/chat/completions; use system instead", + ) + if isinstance(content, list): + # OpenAI multimodal content parts (text/image_url/input_audio/...) are not + # applied by this text-only gateway. Fail closed so SDKs cannot silently + # believe vision/audio parts were processed as plain text. + raise RequestError( + 400, + "invalid_message_content", + "multipart content arrays are not supported on /v1/chat/completions; " + "pass a string content", + ) if not isinstance(role, str) or role not in ALLOWED_MESSAGE_ROLES or not isinstance(content, str): raise RequestError(400, "invalid_message", "message role or content is invalid") - validated.append({"role": role, "content": content}) + # User/system turns drive the prompt — empty content is never applied and + # would only create silent no-op turns. Assistant/tool may still use empty + # content when tool_calls or tool results carry the payload. + if role in {"user", "system"} and not content.strip(): + raise RequestError( + 400, + "invalid_message_content", + "user and system message content must be a non-empty string", + ) + entry: dict[str, str] = {"role": role, "content": content} + if role == "tool": + # OpenAI tool messages bind results to a prior tool_call via tool_call_id. + tool_call_id = message.get("tool_call_id") + if not isinstance(tool_call_id, str) or not tool_call_id.strip(): + raise RequestError( + 400, + "invalid_message", + "tool messages require a non-empty tool_call_id string", + ) + if len(tool_call_id) > 128: + raise RequestError( + 400, + "invalid_message", + "tool_call_id must be at most 128 characters", + ) + entry["tool_call_id"] = tool_call_id + if "name" in message: + # OpenAI optional participant name on system/user/assistant (not tool). + msg_name = message.get("name") + if role == "tool": + raise RequestError( + 400, + "invalid_message_name", + "name is not valid on tool role messages", + ) + if not isinstance(msg_name, str) or not msg_name.strip(): + raise RequestError( + 400, + "invalid_message_name", + "message name must be a non-empty string", + ) + if len(msg_name) > 64: + raise RequestError( + 400, + "invalid_message_name", + "message name must be at most 64 characters", + ) + # OpenAI participant names are alphanumeric plus underscore/hyphen. + if not all(ch.isalnum() or ch in "_-" for ch in msg_name): + raise RequestError( + 400, + "invalid_message_name", + "message name must match [a-zA-Z0-9_-]", + ) + entry["name"] = msg_name + validated.append(entry) + return validated + + +def _validate_chat_tool_message_ids(body: dict[str, Any]) -> None: + """Fail closed on role=tool messages missing a usable tool_call_id. + + Runs before tools passthrough so multi-turn tool results are shape-checked + even when the body is proxied verbatim to a single provider agent. + """ + messages = body.get("messages") + if not isinstance(messages, list): + return + for message in messages: + if not isinstance(message, dict): + continue + if message.get("role") != "tool": + continue + tool_call_id = message.get("tool_call_id") + if not isinstance(tool_call_id, str) or not tool_call_id.strip(): + raise RequestError( + 400, + "invalid_message", + "tool messages require a non-empty tool_call_id string", + ) + if len(tool_call_id) > 128: + raise RequestError( + 400, + "invalid_message", + "tool_call_id must be at most 128 characters", + ) + + +def _validate_chat_assistant_tool_calls(body: dict[str, Any]) -> None: + """OpenAI assistant ``tool_calls`` array shape on chat messages. + + Each entry must be a function tool call with non-empty ``id``, + ``function.name``, and string ``function.arguments`` (JSON text). + Validated before passthrough so multi-turn tool histories fail closed. + """ + messages = body.get("messages") + if not isinstance(messages, list): + return + for message in messages: + if not isinstance(message, dict): + continue + if "tool_calls" not in message: + continue + if message.get("role") != "assistant": + raise RequestError( + 400, + "invalid_message", + "tool_calls is only valid on assistant messages", + ) + tool_calls = message.get("tool_calls") + if not isinstance(tool_calls, list) or not tool_calls: + raise RequestError( + 400, + "invalid_message", + "tool_calls must be a non-empty array", + ) + if len(tool_calls) > 128: + raise RequestError( + 400, + "invalid_message", + "tool_calls must contain at most 128 entries", + ) + for call in tool_calls: + if not isinstance(call, dict): + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry must be an object", + ) + call_id = call.get("id") + if not isinstance(call_id, str) or not call_id.strip(): + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry requires a non-empty id string", + ) + if len(call_id) > 128: + raise RequestError( + 400, + "invalid_message", + "each tool_calls id must be at most 128 characters", + ) + if call.get("type") != "function": + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry type must be function", + ) + function = call.get("function") + if not isinstance(function, dict): + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry requires a function object", + ) + name = function.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.name must be a non-empty string", + ) + if len(name) > 64: + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.name must be at most 64 characters", + ) + if not all(ch.isalnum() or ch in "_-" for ch in name): + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.name must match [a-zA-Z0-9_-]", + ) + arguments = function.get("arguments") + if not isinstance(arguments, str): + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.arguments must be a string", + ) + + +def _validate_openai_metadata(body: dict[str, Any]) -> dict[str, str] | None: + """OpenAI ``metadata`` — object of string pairs, at most 16 entries. + + Keys ≤64 characters; values ≤512 characters. Non-objects and non-string + entries fail closed so clients cannot store untyped junk that cost or + observability consumers would silently drop. + """ + if "metadata" not in body: + return None + metadata = body.get("metadata") + if not isinstance(metadata, dict): + raise RequestError(400, "invalid_metadata", "metadata must be an object") + if len(metadata) > 16: + raise RequestError(400, "invalid_metadata", "metadata must contain at most 16 entries") + validated: dict[str, str] = {} + for key, value in metadata.items(): + if not isinstance(key, str): + raise RequestError(400, "invalid_metadata", "metadata keys must be strings") + if len(key) > 64: + raise RequestError(400, "invalid_metadata", "metadata keys must be at most 64 characters") + if not isinstance(value, str): + raise RequestError(400, "invalid_metadata", "metadata values must be strings") + if len(value) > 512: + raise RequestError( + 400, + "invalid_metadata", + "metadata values must be at most 512 characters", + ) + validated[key] = value return validated @@ -211,6 +1017,12 @@ def _validate_attribution(attribution: Any) -> dict[str, Any] | None: def _validate_routing(routing: Any) -> dict[str, Any] | None: + """OpenAI-adjacent routing hints for sync vs batch channel selection. + + Fail closed on shape so callers cannot smuggle non-boolean latency flags or + free-form priority values that RoutingPolicy would silently misread via + loose coercion (``bool(x)`` / ``str(x)``). + """ if routing is None: return None if not isinstance(routing, dict): @@ -221,6 +1033,22 @@ def _validate_routing(routing: Any) -> dict[str, Any] | None: channel = routing.get("channel") if channel is not None and channel not in {"sync", "batch"}: raise RequestError(400, "invalid_routing", "routing.channel must be sync or batch") + if "latency_tolerant" in routing: + latency_tolerant = routing.get("latency_tolerant") + if not isinstance(latency_tolerant, bool): + raise RequestError( + 400, + "invalid_routing", + "routing.latency_tolerant must be a boolean", + ) + if "priority" in routing: + priority = routing.get("priority") + if not isinstance(priority, str) or priority not in {"interactive", "normal", "bulk"}: + raise RequestError( + 400, + "invalid_routing", + "routing.priority must be one of interactive, normal, bulk", + ) return routing @@ -248,22 +1076,549 @@ def _validate_batch_requests(body: dict[str, Any], expose_trace: bool) -> list[B def _validate_embeddings_inputs(body: dict[str, Any]) -> list[str]: - """Validate the embeddings batch inputs (accepts ``inputs`` or ``input``).""" + """Validate embeddings ``input``/``inputs`` for sync and batch paths. + + Accepts a non-empty string or a non-empty array of non-empty strings. + Blank items fail closed: empty vectors pollute semantic search and cost + rollups without giving buyers a usable meaning unit. + """ raw = body.get("inputs") if raw is None: raw = body.get("input") if isinstance(raw, str): raw = [raw] if not isinstance(raw, list) or not raw: - raise RequestError(400, "invalid_request", "input/inputs must be a non-empty array of strings") + raise RequestError( + 400, + "invalid_input", + "input/inputs must be a non-empty string or non-empty array of strings", + ) inputs: list[str] = [] for item in raw: if not isinstance(item, str): - raise RequestError(400, "invalid_request", "each embedding input must be a string") + raise RequestError(400, "invalid_input", "each embedding input must be a string") + if not item.strip(): + raise RequestError( + 400, + "invalid_input", + "each embedding input must be a non-empty string", + ) inputs.append(item) return inputs + +def _validate_chat_store(body: dict[str, Any]) -> bool | None: + """Chat Completions ``store`` — strict boolean; ``true`` is not supported. + + OpenAI can persist completions when ``store=true``. This gateway does not + implement that persistence surface, so ``store=true`` fails closed. + ``store=false`` and omit remain valid (explicit no-store is honest). + """ + if "store" not in body: + return None + store = body.get("store") + if not isinstance(store, bool): + raise RequestError(400, "invalid_store", "store must be a boolean") + if store is True: + raise RequestError( + 400, + "invalid_store", + "store=true is not supported on /v1/chat/completions", + ) + return store + + + + +def _validate_chat_reasoning_effort(body: dict[str, Any]) -> None: + """Chat Completions ``reasoning_effort`` — not applied on multi-agent route. + + OpenAI o-series models accept ``reasoning_effort`` (e.g. none/low/medium/high). + This gateway never threads the knob into ``ModelClient`` on the orchestration + path, so any present value fails closed rather than silently ignoring a + buyer-visible reasoning control. + """ + if "reasoning_effort" not in body: + return + raise RequestError( + 400, + "invalid_reasoning_effort", + "reasoning_effort is not supported on /v1/chat/completions", + ) + + + +def _validate_chat_modalities(body: dict[str, Any]) -> list[str] | None: + """Chat Completions ``modalities`` — omit or ``["text"]`` only. + + OpenAI selects output types (text/audio) via modalities. This gateway is + text-only; non-text modalities fail closed so clients cannot silently + believe audio (or other) output was applied. + """ + if "modalities" not in body: + return None + modalities = body.get("modalities") + if not isinstance(modalities, list) or not modalities: + raise RequestError( + 400, + "invalid_modalities", + "modalities must be a non-empty array of strings", + ) + if any(not isinstance(item, str) for item in modalities): + raise RequestError( + 400, + "invalid_modalities", + "modalities must be a non-empty array of strings", + ) + if modalities != ["text"]: + raise RequestError( + 400, + "invalid_modalities", + 'only modalities ["text"] is supported on /v1/chat/completions', + ) + return modalities + + +def _validate_chat_prediction(body: dict[str, Any]) -> None: + """Chat Completions ``prediction`` (Predicted Outputs) — not supported. + + OpenAI Predicted Outputs lets clients supply expected completion content for + latency wins. This gateway does not apply ``prediction`` on the multi-agent + route path, so any present value fails closed rather than silently ignoring + a buyer-visible optimization hint. + """ + if "prediction" not in body: + return + raise RequestError( + 400, + "invalid_prediction", + "prediction is not supported on /v1/chat/completions", + ) + + +def _validate_chat_response_format(body: dict[str, Any]) -> dict[str, Any] | None: + """OpenAI chat ``response_format`` — object with type text/json_object/json_schema. + + Shape is validated before passthrough so malformed payloads fail closed + rather than reaching a provider with an unusable format object. + + OpenAI type-only forms are strict: ``text`` and ``json_object`` accept only + the ``type`` key. ``json_schema`` accepts only ``type`` and ``json_schema``. + Extra sibling keys fail closed so clients cannot smuggle unsupported fields + into a provider-shaped object that this gateway never interpreted. + """ + if "response_format" not in body: + return None + fmt = body.get("response_format") + if not isinstance(fmt, dict): + raise RequestError( + 400, + "invalid_response_format", + "response_format must be an object", + ) + fmt_type = fmt.get("type") + if fmt_type not in ("text", "json_object", "json_schema"): + raise RequestError( + 400, + "invalid_response_format", + "response_format.type must be one of text, json_object, json_schema", + ) + if fmt_type in ("text", "json_object"): + # OpenAI: {"type": "json_object"} / {"type": "text"} — no siblings. + unknown = sorted(set(fmt) - {"type"}) + if unknown: + raise RequestError( + 400, + "invalid_response_format", + f"response_format with type {fmt_type} accepts only the type field", + {"fields": unknown}, + ) + return fmt + if fmt_type == "json_schema": + unknown = sorted(set(fmt) - {"type", "json_schema"}) + if unknown: + raise RequestError( + 400, + "invalid_response_format", + "response_format with type json_schema accepts only type and json_schema", + {"fields": unknown}, + ) + schema = fmt.get("json_schema") + if not isinstance(schema, dict): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema must be an object when type is json_schema", + ) + name = schema.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.name must be a non-empty string", + ) + # OpenAI requires json_schema.schema as the actual JSON Schema object. + # Fail closed when missing or non-object so clients cannot silently + # believe structured-output enforcement applied without a schema body. + schema_body = schema.get("schema") + if not isinstance(schema_body, dict): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.schema must be an object", + ) + if "strict" in schema and not isinstance(schema.get("strict"), bool): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.strict must be a boolean when provided", + ) + return fmt + + + +def _validate_chat_tools(body: dict[str, Any]) -> list[dict[str, Any]] | None: + """OpenAI chat ``tools`` — non-empty array of function tool objects. + + Each entry must be an object with ``type`` == ``function`` and a + ``function`` object that has a non-empty ``name``. Shape-only validation + before passthrough; provider schema depth is not re-checked here. + """ + if "tools" not in body: + return None + tools = body.get("tools") + if not isinstance(tools, list) or not tools: + raise RequestError( + 400, + "invalid_tools", + "tools must be a non-empty array", + ) + if len(tools) > 128: + raise RequestError( + 400, + "invalid_tools", + "tools must contain at most 128 entries", + ) + validated: list[dict[str, Any]] = [] + for item in tools: + if not isinstance(item, dict): + raise RequestError(400, "invalid_tools", "each tool must be an object") + # OpenAI tool objects are type + function only; extra siblings fail closed + # so clients cannot smuggle uninterpreted fields through passthrough. + unknown_tool = sorted(set(item) - {"type", "function"}) + if unknown_tool: + raise RequestError( + 400, + "invalid_tools", + "each tool accepts only type and function fields", + {"fields": unknown_tool}, + ) + if item.get("type") != "function": + raise RequestError( + 400, + "invalid_tools", + "each tool type must be function", + ) + function = item.get("function") + if not isinstance(function, dict): + raise RequestError( + 400, + "invalid_tools", + "each tool.function must be an object", + ) + unknown_fn = sorted(set(function) - {"name", "description", "parameters", "strict"}) + if unknown_fn: + raise RequestError( + 400, + "invalid_tools", + "each tool.function accepts only name, description, parameters, and strict", + {"fields": unknown_fn}, + ) + if "strict" in function and not isinstance(function.get("strict"), bool): + raise RequestError( + 400, + "invalid_tools", + "each tool.function.strict must be a boolean when provided", + ) + name = function.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_tools", + "each tool.function.name must be a non-empty string", + ) + # OpenAI function names: [a-zA-Z0-9_-]{1,64} + if len(name) > 64: + raise RequestError( + 400, + "invalid_tools", + "each tool.function.name must be at most 64 characters", + ) + if not all(ch.isalnum() or ch in "_-" for ch in name): + raise RequestError( + 400, + "invalid_tools", + "each tool.function.name must match [a-zA-Z0-9_-]", + ) + # OpenAI function tools require parameters as a JSON Schema object when present. + if "parameters" in function: + parameters = function.get("parameters") + if not isinstance(parameters, dict): + raise RequestError( + 400, + "invalid_tools", + "each tool.function.parameters must be an object", + ) + if "description" in function and not isinstance(function.get("description"), str): + raise RequestError( + 400, + "invalid_tools", + "each tool.function.description must be a string when provided", + ) + validated.append(item) + return validated + + +def _validate_chat_tool_choice(body: dict[str, Any]) -> str | dict[str, Any] | None: + """OpenAI chat ``tool_choice`` — none/auto/required or named function object. + + When ``type`` is ``function``, ``function.name`` must match a tools entry + so clients cannot force a tool the request did not declare. + """ + if "tool_choice" not in body: + return None + choice = body.get("tool_choice") + if isinstance(choice, str): + if choice not in ("none", "auto", "required"): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice string must be one of none, auto, required", + ) + return choice + if isinstance(choice, dict): + # OpenAI named tool_choice is {type, function}; extra siblings fail closed. + unknown = sorted(set(choice) - {"type", "function"}) + if unknown: + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice object accepts only type and function fields", + {"fields": unknown}, + ) + if choice.get("type") != "function": + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice object type must be function", + ) + function = choice.get("function") + if not isinstance(function, dict): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function must be an object with a name", + ) + unknown_fn = sorted(set(function) - {"name"}) + if unknown_fn: + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function accepts only name", + {"fields": unknown_fn}, + ) + name = function.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function.name must be a non-empty string", + ) + tools = body.get("tools") + tool_names: set[str] = set() + if isinstance(tools, list): + for item in tools: + if not isinstance(item, dict): + continue + fn = item.get("function") + if isinstance(fn, dict): + tool_name = fn.get("name") + if isinstance(tool_name, str): + tool_names.add(tool_name) + if name not in tool_names: + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function.name must match a tools entry", + ) + return choice + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice must be a string or object", + ) + + + + + + +def _validate_responses_model(body: dict[str, Any]) -> str: + """Responses API ``model`` — required non-empty string ≤256 chars. + + OpenAI requires model on Responses. Missing/empty/non-string values fail + closed so clients cannot hit passthrough with an implicit mock default and + believe a named deployment was selected. + """ + model = body.get("model") + if model is None: + raise RequestError(400, "invalid_model", "model is required on /v1/responses") + if not isinstance(model, str) or not model.strip(): + raise RequestError(400, "invalid_model", "model must be a non-empty string") + if len(model) > 256: + raise RequestError(400, "invalid_model", "model must be at most 256 characters") + return model + + +def _validate_responses_instructions(body: dict[str, Any]) -> str | None: + """Responses API ``instructions`` — optional non-empty string ≤32000 chars. + + OpenAI system-style instructions for the Responses surface. Empty strings + and non-strings fail closed so clients cannot ship a silent no-op that + looks like a configured system prompt. + """ + if "instructions" not in body: + return None + value = body.get("instructions") + if not isinstance(value, str): + raise RequestError(400, "invalid_instructions", "instructions must be a string") + if not value.strip(): + raise RequestError( + 400, + "invalid_instructions", + "instructions must be a non-empty string on /v1/responses", + ) + if len(value) > 32_000: + raise RequestError( + 400, + "invalid_instructions", + "instructions must be at most 32000 characters", + ) + return value + + +def _validate_responses_reasoning(body: dict[str, Any]) -> None: + """Responses API ``reasoning`` — not applied on single-agent passthrough. + + OpenAI Responses accepts a ``reasoning`` object (effort/summary controls). + This gateway proxies Responses but does not interpret or enforce reasoning + controls, so any present value fails closed rather than silently ignoring a + buyer-visible o-series control surface. + """ + if "reasoning" not in body: + return + raise RequestError( + 400, + "invalid_reasoning", + "reasoning is not supported on /v1/responses", + ) + + + +def _validate_batch_embeddings_endpoint(body: dict[str, Any]) -> str | None: + """Batch embeddings ``endpoint`` — optional non-empty string alias ≤256 chars. + + naruon and OpenAI-compatible clients may tag the upstream embeddings route + (e.g. ``/v1/embeddings``). Empty/null/non-string values fail closed so the + gateway never records a blank endpoint alias as if a route was selected. + """ + if "endpoint" not in body: + return None + value = body.get("endpoint") + if not isinstance(value, str) or not value.strip(): + raise RequestError( + 400, + "invalid_endpoint", + "endpoint must be a non-empty string on /v1/batch/embeddings", + ) + if len(value) > 256: + raise RequestError( + 400, + "invalid_endpoint", + "endpoint must be at most 256 characters", + ) + return value + + +def _validate_embeddings_model(body: dict[str, Any]) -> str: + """OpenAI embeddings ``model`` — required non-empty string ≤256 chars.""" + model = body.get("model") + if model is None: + raise RequestError(400, "invalid_model", "model is required") + if not isinstance(model, str) or not model.strip(): + raise RequestError(400, "invalid_model", "model must be a non-empty string") + if len(model) > 256: + raise RequestError(400, "invalid_model", "model must be at most 256 characters") + return model + + +def _validate_embeddings_encoding_format(body: dict[str, Any]) -> str | None: + """OpenAI ``encoding_format`` — omit or ``float`` only; base64 fail-closed. + + This gateway returns float vectors on the OpenAI list shape. ``base64`` is + not produced, so requesting it fails closed rather than silently returning + floats. + """ + if "encoding_format" not in body: + return None + value = body.get("encoding_format") + if not isinstance(value, str): + raise RequestError(400, "invalid_encoding_format", "encoding_format must be a string") + if value != "float": + raise RequestError( + 400, + "invalid_encoding_format", + 'only encoding_format "float" is supported on /v1/embeddings', + ) + return value + + +def _validate_embeddings_dimensions(body: dict[str, Any]) -> None: + """OpenAI ``dimensions`` — not applied on this gateway; any value fails closed.""" + if "dimensions" not in body: + return + raise RequestError( + 400, + "invalid_dimensions", + "dimensions is not supported on /v1/embeddings", + ) + + +def _openai_embeddings_response(document: dict[str, Any], *, model: str) -> dict[str, Any]: + """Map batch document vectors to the OpenAI ``/v1/embeddings`` list shape.""" + items = document.get("embeddings") or [] + data = [] + for item in items: + data.append( + { + "object": "embedding", + "index": int(item.get("index", 0)), + "embedding": list(item.get("embedding") or []), + } + ) + total_tokens = int(document.get("total_tokens") or 0) + return { + "object": "list", + "data": data, + "model": model or document.get("model") or "contextual-orchestrator", + "usage": { + "prompt_tokens": total_tokens, + "total_tokens": total_tokens, + }, + } + + def _embeddings_attribution(body: dict[str, Any]) -> dict[str, Any]: """Build ledger attribution from the explicit ``attribution`` field merged with any attribution dimensions carried inside ``metadata``. @@ -711,8 +2066,153 @@ def do_POST(self) -> None: # noqa: N802 self._send(orchestrator.add_agent(segments[3], body), 201) return + if path == "/v1/completions": + # Legacy OpenAI Completions: prompt → route → text_completion. + _reject_unknown_keys(body, ALLOWED_COMPLETIONS_KEYS) + _validate_completions_stream(body) + _validate_completions_stream_options(body) + _validate_completions_best_of(body) + _validate_completions_echo(body) + _validate_completions_suffix(body) + _validate_completions_logprobs(body) + max_tokens = _validate_completions_max_tokens(body) + model_name = _validate_completions_model(body) + _require_pool_model(orchestrator, model_name) + top_p = _validate_completions_top_p(body) + temperature = _validate_completions_temperature(body) + presence_penalty = _validate_completions_presence_penalty(body) + frequency_penalty = _validate_completions_frequency_penalty(body) + _validate_completions_seed(body) + _validate_completions_stop(body) + _validate_completions_n(body) + end_user_id = _validate_completions_user(body) + _validate_completions_logit_bias(body) + _validate_service_tier(body, endpoint_path="/v1/completions") + if "metadata" in body: + _validate_openai_metadata(body) + if "prompt" not in body: + raise RequestError(400, "invalid_prompt", "prompt is required") + messages = _validate_completion_prompt(body.get("prompt")) + attribution = _validate_attribution(body.get("attribution")) + attribution = dict(attribution or {}) + # OpenAI ``user`` → cost-ledger account when attribution.account is unset. + if end_user_id is not None and not attribution.get("account"): + attribution["account"] = end_user_id + # Request model id → model_name dimension when unset (cost rollups). + if model_name and not attribution.get("model_name"): + attribution["model_name"] = model_name + # Endpoint product surface → service dimension when unset. + if not attribution.get("service"): + attribution["service"] = "completions_api" + routing = _validate_routing(body.get("routing")) + started_at = time.perf_counter() + # Apply request sampling knobs to the provider client for this call. + model_client = orchestrator.client + previous_max_tokens = model_client.max_output_tokens + previous_temperature = model_client.default_temperature + previous_top_p = model_client.default_top_p + previous_presence = model_client.default_presence_penalty + previous_frequency = model_client.default_frequency_penalty + if max_tokens is not None: + model_client.max_output_tokens = max_tokens + if temperature is not None: + model_client.default_temperature = temperature + if top_p is not None: + model_client.default_top_p = top_p + if presence_penalty is not None: + model_client.default_presence_penalty = presence_penalty + if frequency_penalty is not None: + model_client.default_frequency_penalty = frequency_penalty + try: + result = self._run(lambda: coordinator.complete( + messages, + mode="route", + attribution=attribution, + hints=routing, + model_name=model_name, + workflow_run_id=f"run_{uuid.uuid4().hex}", + )) + finally: + model_client.max_output_tokens = previous_max_tokens + model_client.default_temperature = previous_temperature + model_client.default_top_p = previous_top_p + model_client.default_presence_penalty = previous_presence + model_client.default_frequency_penalty = previous_frequency + # Batch-channel Completions return a job handle (202), not a + # text_completion body — match chat Completions honesty so + # clients never receive a 500 on a valid batch routing hint. + if isinstance(result, dict) and result.get("channel") == "batch": + orchestrator.record_analytics_event( + "text_completion_batched", + { + "endpoint_path": "/v1/completions", + "actor_scope": "inference", + "status_code": 202, + "batch_job_id": result.get("job_id"), + "batch_backend": result.get("backend"), + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + }, + ) + self._send(result, 202) + return + orchestrator.record_analytics_event( + "text_completion_requested", + { + "endpoint_path": "/v1/completions", + "actor_scope": "inference", + "status_code": 200, + "run_mode": "route", + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + }, + ) + self._send(text_completion_response( + result, model=model_name, usage=result.get("usage"), + )) + return if path == "/v1/chat/completions": _reject_unknown_keys(body, ALLOWED_CHAT_KEYS) + if "functions" in body or "function_call" in body: + # OpenAI deprecated functions/function_call in favor of tools/tool_choice. + # Fail closed with a migration message rather than silent passthrough of + # a deprecated surface clients may still send from old SDKs. + raise RequestError( + 400, + "invalid_functions", + "functions and function_call are not supported on /v1/chat/completions; " + "use tools and tool_choice instead", + ) + if "tool_choice" in body and "tools" not in body: + # tool_choice alone is invalid without tools definitions. + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice requires tools on /v1/chat/completions", + ) + # Shape-check tool results before passthrough or orchestration. + _validate_chat_tool_message_ids(body) + _validate_chat_assistant_tool_calls(body) + if "response_format" in body: + _validate_chat_response_format(body) + if "tools" in body: + _validate_chat_tools(body) + if "tool_choice" in body: + _validate_chat_tool_choice(body) + if "parallel_tool_calls" in body: + # Always type-check. With tools, true/false both valid for + # provider passthrough; without tools, true fails closed. + ptc = body.get("parallel_tool_calls") + if not isinstance(ptc, bool): + raise RequestError( + 400, + "invalid_parallel_tool_calls", + "parallel_tool_calls must be a boolean", + ) + if ptc is True and "tools" not in body: + raise RequestError( + 400, + "invalid_parallel_tool_calls", + "parallel_tool_calls=true requires tools on /v1/chat/completions", + ) if PASSTHROUGH_TRIGGER_KEYS & set(body): # response_format / tools cannot be merged across agents; # proxy the full request to one agent and return it verbatim. @@ -733,36 +2233,186 @@ def do_POST(self) -> None: # noqa: N802 return messages = _validate_messages(body.get("messages")) mode = _validate_mode(body.get("orchestration") or body.get("orchestration_mode") or body.get("mode") or "auto") - include_trace = bool(body.get("include_orchestration_trace", security.expose_trace_by_default)) + if "include_orchestration_trace" in body: + include_trace_raw = body.get("include_orchestration_trace") + if not isinstance(include_trace_raw, bool): + raise RequestError( + 400, + "invalid_include_orchestration_trace", + "include_orchestration_trace must be a boolean", + ) + include_trace = include_trace_raw + else: + include_trace = bool(security.expose_trace_by_default) stream = body.get("stream", False) if not isinstance(stream, bool): raise RequestError(400, "invalid_request", "stream must be a boolean") + if "stream_options" in body: + _validate_chat_stream_options(body, stream) attribution = _validate_attribution(body.get("attribution")) routing = _validate_routing(body.get("routing")) - model_name = str(body.get("model", "contextual-orchestrator")) - started_at = time.perf_counter() - if stream and orchestrator.would_route(messages, mode): - self._stream_route_completion(orchestrator, security, messages, model_name) - orchestrator.record_analytics_event( - "chat_completion_requested", - { - "endpoint_path": "/v1/chat/completions", - "actor_scope": "inference", - "status_code": 200, - "run_mode": "route", - "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), - "response_streamed": True, - }, + # Require model — silent default to contextual-orchestrator hid + # which deployment the buyer selected on the chat Completions path. + model_name = _validate_completions_model(body) + _require_pool_model(orchestrator, model_name) + attribution = dict(attribution or {}) + # OpenAI chat ``user`` → account when unset. + # Same fail-closed rules as Completions: present key must be a + # non-empty string ≤64 chars (null/empty/non-string rejected). + end_user_id = _validate_completions_user(body) + if end_user_id is not None and not attribution.get("account"): + attribution["account"] = end_user_id + if model_name and not attribution.get("model_name"): + attribution["model_name"] = model_name + if not attribution.get("service"): + attribution["service"] = "chat_completions_api" + temperature = None + top_p = None + max_tokens = None + presence_penalty = None + frequency_penalty = None + if "temperature" in body: + temperature = _validate_completions_temperature(body) + if "top_p" in body: + top_p = _validate_completions_top_p(body) + # OpenAI: max_completion_tokens takes precedence over max_tokens. + if "max_completion_tokens" in body: + max_tokens = _validate_chat_max_completion_tokens(body) + elif "max_tokens" in body: + max_tokens = _validate_completions_max_tokens(body) + if "presence_penalty" in body: + presence_penalty = _validate_completions_presence_penalty(body) + if "frequency_penalty" in body: + frequency_penalty = _validate_completions_frequency_penalty(body) + if "seed" in body: + # Type-check then fail closed: chat route does not apply seed. + _validate_completions_seed(body) + raise RequestError( + 400, + "invalid_seed", + "seed is not supported on /v1/chat/completions", ) - return - result = self._run(lambda: coordinator.complete( - messages, - mode=mode, - attribution=attribution, - hints=routing, - model_name=model_name, - workflow_run_id=f"run_{uuid.uuid4().hex}", - )) + if "logit_bias" in body: + # Empty {} is an honest no-op (shared Completions helper). + # Non-empty maps fail closed with a chat-path message. + try: + _validate_completions_logit_bias(body) + except RequestError as exc: + if ( + exc.code == "invalid_logit_bias" + and "not supported" in exc.message + ): + raise RequestError( + 400, + "invalid_logit_bias", + "logit_bias is not supported on /v1/chat/completions", + ) from exc + raise + if "stop" in body: + try: + _validate_completions_stop(body) + except RequestError as exc: + # Completions helper fails closed with a Completions path message; + # re-surface for chat with the chat endpoint string. + if exc.code == "invalid_stop" and "not supported" in exc.message: + raise RequestError( + 400, + "invalid_stop", + "stop sequences are not supported on /v1/chat/completions", + ) from exc + raise + raise RequestError( + 400, + "invalid_stop", + "stop sequences are not supported on /v1/chat/completions", + ) + if "n" in body: + try: + _validate_completions_n(body) + except RequestError as exc: + if exc.code == "invalid_n" and "not supported" in exc.message: + raise RequestError( + 400, + "invalid_n", + "n greater than 1 is not supported on /v1/chat/completions", + ) from exc + raise + if "logprobs" in body or "top_logprobs" in body: + # Chat route path does not return token logprobs; fail closed. + if "logprobs" in body: + lp = body.get("logprobs") + if not isinstance(lp, bool): + raise RequestError(400, "invalid_logprobs", "logprobs must be a boolean") + if lp is True: + raise RequestError( + 400, + "invalid_logprobs", + "logprobs=true is not supported on /v1/chat/completions", + ) + if "top_logprobs" in body: + raise RequestError( + 400, + "invalid_top_logprobs", + "top_logprobs is not supported on /v1/chat/completions", + ) + if "store" in body: + _validate_chat_store(body) + if "modalities" in body: + _validate_chat_modalities(body) + if "prediction" in body: + _validate_chat_prediction(body) + if "reasoning_effort" in body: + _validate_chat_reasoning_effort(body) + if "service_tier" in body: + _validate_service_tier(body, endpoint_path="/v1/chat/completions") + if "metadata" in body: + _validate_openai_metadata(body) + started_at = time.perf_counter() + model_client = orchestrator.client + previous_max_tokens = model_client.max_output_tokens + previous_temperature = model_client.default_temperature + previous_top_p = model_client.default_top_p + previous_presence = model_client.default_presence_penalty + previous_frequency = model_client.default_frequency_penalty + if max_tokens is not None: + model_client.max_output_tokens = max_tokens + if temperature is not None: + model_client.default_temperature = temperature + if top_p is not None: + model_client.default_top_p = top_p + if presence_penalty is not None: + model_client.default_presence_penalty = presence_penalty + if frequency_penalty is not None: + model_client.default_frequency_penalty = frequency_penalty + try: + if stream and orchestrator.would_route(messages, mode): + self._stream_route_completion(orchestrator, security, messages, model_name) + orchestrator.record_analytics_event( + "chat_completion_requested", + { + "endpoint_path": "/v1/chat/completions", + "actor_scope": "inference", + "status_code": 200, + "run_mode": "route", + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + "response_streamed": True, + }, + ) + return + result = self._run(lambda: coordinator.complete( + messages, + mode=mode, + attribution=attribution, + hints=routing, + model_name=model_name, + workflow_run_id=f"run_{uuid.uuid4().hex}", + )) + finally: + model_client.max_output_tokens = previous_max_tokens + model_client.default_temperature = previous_temperature + model_client.default_top_p = previous_top_p + model_client.default_presence_penalty = previous_presence + model_client.default_frequency_penalty = previous_frequency # Latency-tolerant requests get dispatched to the batch backend. if result.get("channel") == "batch": orchestrator.record_analytics_event( @@ -797,15 +2447,91 @@ def do_POST(self) -> None: # noqa: N802 result, model=model_name, include_trace=include_trace, usage=result.get("usage"), )) return + if path == "/v1/embeddings": + # OpenAI sync embeddings: input → vectors as list object. + # Reuses the embedding batch backend (local path completes + # synchronously) and frames an OpenAI-shaped response so + # SDKs that call /v1/embeddings work without the batch path. + _reject_unknown_keys(body, ALLOWED_EMBEDDINGS_KEYS) + model_name = _validate_embeddings_model(body) + _validate_embeddings_encoding_format(body) + _validate_embeddings_dimensions(body) + end_user_id = _validate_completions_user(body) + if "metadata" in body and not isinstance(body.get("metadata"), dict): + # OpenAI-shaped string metadata is preferred for this + # surface; non-objects fail closed before attribution merge. + raise RequestError(400, "invalid_metadata", "metadata must be an object") + if "metadata" in body: + # When all values are strings, enforce OpenAI ≤16 pairs; + # naruon-style attribution-in-metadata still uses + # _embeddings_attribution below for known dimensions. + meta = body.get("metadata") or {} + if meta and all(isinstance(v, str) for v in meta.values()): + _validate_openai_metadata(body) + if "input" not in body and "inputs" not in body: + # OpenAI only documents ``input``; accept nothing else. + raise RequestError(400, "invalid_input", "input is required on /v1/embeddings") + # Prefer OpenAI ``input``; do not accept ``inputs`` on this path + # (batch endpoint owns ``inputs``) so clients get a clear split. + if "inputs" in body and "input" not in body: + raise RequestError( + 400, + "invalid_input", + "use input on /v1/embeddings; inputs is only for /v1/batch/embeddings", + ) + inputs = _validate_embeddings_inputs({"input": body.get("input")}) + attribution = _embeddings_attribution(body) + attribution = dict(attribution or {}) + if end_user_id is not None and not attribution.get("account"): + attribution["account"] = end_user_id + if model_name and not attribution.get("model_name"): + attribution["model_name"] = model_name + if not attribution.get("service"): + attribution["service"] = "embeddings_api" + started_at = time.perf_counter() + document = self._run(lambda: coordinator.complete_embeddings_batch( + inputs, + model=model_name, + attribution=attribution, + metadata={"actor_scope": "inference", "endpoint_alias": "embeddings"}, + )) + if document.get("status") != "completed" or document.get("embeddings") is None: + # Async backends return a job handle; fail closed on the + # sync OpenAI path rather than inventing vectors. + raise RequestError( + 503, + "embeddings_unavailable", + "sync /v1/embeddings is unavailable for this backend; use /v1/batch/embeddings", + ) + orchestrator.record_analytics_event( + "embeddings_requested", + { + "endpoint_path": "/v1/embeddings", + "actor_scope": "inference", + "status_code": 200, + "input_count": len(inputs), + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + }, + ) + self._send(_openai_embeddings_response(document, model=model_name)) + return if path == "/v1/batch/embeddings": _reject_unknown_keys(body, ALLOWED_EMBEDDINGS_BATCH_KEYS) inputs = _validate_embeddings_inputs(body) - model_name = str(body.get("model", "contextual-orchestrator")) + # Require model — silent default to contextual-orchestrator was an + # honesty gap for naruon/batch clients that omit the field. + if "model" not in body: + raise RequestError( + 400, + "invalid_model", + "model is required on /v1/batch/embeddings", + ) + model_name = _validate_embeddings_model(body) attribution = _embeddings_attribution(body) submit_metadata: dict[str, Any] = {"actor_scope": "inference"} - endpoint_alias = body.get("endpoint") - if endpoint_alias: - submit_metadata["endpoint_alias"] = str(endpoint_alias) + endpoint_alias = _validate_batch_embeddings_endpoint(body) + if endpoint_alias is not None: + submit_metadata["endpoint_alias"] = endpoint_alias document = self._run(lambda: coordinator.complete_embeddings_batch( inputs, model=model_name, @@ -862,6 +2588,38 @@ def do_POST(self) -> None: # noqa: N802 # The Responses API has no chat-completions verifier equivalent, # so every request is proxied to one agent verbatim. _reject_unknown_keys(body, ALLOWED_RESPONSES_KEYS) + # Fail-closed shape checks before passthrough so buyers never + # get a 200 after shipping invalid OpenAI-shaped metadata/input. + _validate_responses_model(body) + if "reasoning" in body: + _validate_responses_reasoning(body) + if "instructions" in body: + _validate_responses_instructions(body) + if "metadata" in body: + _validate_openai_metadata(body) + if "input" not in body: + raise RequestError(400, "invalid_input", "input is required on /v1/responses") + input_value = body.get("input") + if not isinstance(input_value, (str, list)) or ( + isinstance(input_value, str) and not input_value.strip() + ) or (isinstance(input_value, list) and len(input_value) == 0): + raise RequestError( + 400, + "invalid_input", + "input must be a non-empty string or non-empty array on /v1/responses", + ) + # stream=false / omit → non-SSE JSON response (honest no-stream path). + # stream=true is not implemented for Responses passthrough. + if "stream" in body: + stream = body.get("stream") + if not isinstance(stream, bool): + raise RequestError(400, "invalid_stream", "stream must be a boolean") + if stream is True: + raise RequestError( + 400, + "invalid_stream", + "stream is not supported on /v1/responses", + ) started_at = time.perf_counter() proxied = self._run( lambda: orchestrator.proxy_completion(body, endpoint="responses") diff --git a/tests/test_analytics_runtime.py b/tests/test_analytics_runtime.py index 5fad97af6..83c77888f 100644 --- a/tests/test_analytics_runtime.py +++ b/tests/test_analytics_runtime.py @@ -114,7 +114,7 @@ def test_analytics_endpoint_and_admin_console_use_source_backed_snapshot() -> No try: chat_status, _ = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "hello"}]}, + {"model": "mock-planner", "messages": [{"role": "user", "content": "hello"}]}, "secret_token", ) snapshot_status, snapshot = get_json( diff --git a/tests/test_budget_enforcement.py b/tests/test_budget_enforcement.py index 6ec0816b4..3f3003a8c 100644 --- a/tests/test_budget_enforcement.py +++ b/tests/test_budget_enforcement.py @@ -86,7 +86,7 @@ def test_http_over_budget_returns_429() -> None: port = server.server_address[1] request = urllib.request.Request( f"http://127.0.0.1:{port}/v1/chat/completions", - data=json.dumps({"messages": [{"role": "user", "content": "blocked"}]}).encode("utf-8"), + data=json.dumps({"model": "test-model", "messages": [{"role": "user", "content": "blocked"}]}).encode("utf-8"), headers={"content-type": "application/json", "authorization": f"Bearer {token}", "connection": "close"}, method="POST", ) diff --git a/tests/test_chat_developer_multimodal_content_http_honesty.py b/tests/test_chat_developer_multimodal_content_http_honesty.py new file mode 100644 index 000000000..f2ba09740 --- /dev/null +++ b/tests/test_chat_developer_multimodal_content_http_honesty.py @@ -0,0 +1,171 @@ +"""Chat message content honesty: developer role and multimodal arrays fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_message_content_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_string_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "plain text invoice note"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_developer_role() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "developer", "content": "system-like instructions"}, + {"role": "user", "content": "hi"}, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_role" in blob + assert "developer" in blob + assert "system" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_multipart_image_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this receipt"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/receipt.png"}, + }, + ], + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_content" in blob + assert "multipart" in blob or "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_input_audio_content_part() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": {"data": "AAAA", "format": "wav"}, + } + ], + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_non_string_non_array_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": 12345}], + }, + ) + assert status == 400, body + assert "invalid_message_content" in json.dumps(body) or "invalid_message" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_string_content() + test_http_chat_rejects_developer_role() + test_http_chat_rejects_multipart_image_content() + test_http_chat_rejects_input_audio_content_part() + test_http_chat_rejects_non_string_non_array_content() + print("ok") diff --git a/tests/test_chat_empty_user_system_content_http_honesty.py b/tests/test_chat_empty_user_system_content_http_honesty.py new file mode 100644 index 000000000..44398dc34 --- /dev/null +++ b/tests/test_chat_empty_user_system_content_http_honesty.py @@ -0,0 +1,154 @@ +"""Chat user/system empty content fail-closed; assistant empty allowed without tools.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_empty_user_system_content_http_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_empty_user_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": " "}], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_content" in blob + assert "non-empty" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_system_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "system", "content": ""}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_nonempty_system_and_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "system", "content": "You are a finance clerk."}, + {"role": "user", "content": "Draft a receipt line."}, + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_assistant_content_in_history() -> None: + """Assistant turns may carry empty content when history only needs the role turn.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": ""}, + {"role": "user", "content": "continue please"}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_unknown_role() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "owner", "content": "hi"}], + }, + ) + assert status == 400, body + assert "invalid_message" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_empty_user_content() + test_http_chat_rejects_empty_system_content() + test_http_chat_accepts_nonempty_system_and_user() + test_http_chat_accepts_empty_assistant_content_in_history() + test_http_chat_rejects_unknown_role() + print("ok") diff --git a/tests/test_chat_include_orchestration_trace_http_honesty.py b/tests/test_chat_include_orchestration_trace_http_honesty.py new file mode 100644 index 000000000..006e4288d --- /dev/null +++ b/tests/test_chat_include_orchestration_trace_http_honesty.py @@ -0,0 +1,154 @@ +"""Chat Completions include_orchestration_trace honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_include_orchestration_trace_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + # expose_trace_by_default false so omit hides trace unless request opts in. + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, expose_trace_by_default=False), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_include_orchestration_trace_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace string"}], + "include_orchestration_trace": "yes", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_include_orchestration_trace" in blob + assert "boolean" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_include_orchestration_trace_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace null"}], + "include_orchestration_trace": None, + }, + ) + assert status == 400, body + assert "invalid_include_orchestration_trace" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_include_orchestration_trace_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace on"}], + "include_orchestration_trace": True, + }, + ) + assert status == 200, body + # Opt-in must surface orchestration for trusted callers. + assert "orchestration" in body or "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_include_orchestration_trace_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace off"}], + "include_orchestration_trace": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_include_orchestration_trace_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no trace flag"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_include_orchestration_trace_non_boolean() + test_http_chat_rejects_include_orchestration_trace_null() + test_http_chat_accepts_include_orchestration_trace_true() + test_http_chat_accepts_include_orchestration_trace_false() + test_http_chat_accepts_include_orchestration_trace_omitted() + print("ok") diff --git a/tests/test_chat_modalities_http_honesty.py b/tests/test_chat_modalities_http_honesty.py new file mode 100644 index 000000000..a7908997b --- /dev/null +++ b/tests/test_chat_modalities_http_honesty.py @@ -0,0 +1,167 @@ +"""Chat Completions modalities honesty over HTTP (text-only gateway).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_modalities_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_modalities_text_only() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "text only"}], + "modalities": ["text"], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_modalities_audio() -> None: + """Buyers must not believe audio output was produced by a text-only gateway.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "speak"}], + "modalities": ["audio"], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_modalities" in blob + assert "text" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_modalities_text_and_audio() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "both"}], + "modalities": ["text", "audio"], + }, + ) + assert status == 400, body + assert "invalid_modalities" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_modalities() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty mods"}], + "modalities": [], + }, + ) + assert status == 400, body + assert "invalid_modalities" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_modalities_non_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "string mods"}], + "modalities": "text", + }, + ) + assert status == 400, body + assert "invalid_modalities" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_modalities_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "default modalities"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_modalities_text_only() + test_http_chat_rejects_modalities_audio() + test_http_chat_rejects_modalities_text_and_audio() + test_http_chat_rejects_empty_modalities() + test_http_chat_rejects_modalities_non_array() + test_http_chat_accepts_modalities_omitted() + print("ok") diff --git a/tests/test_chat_openai_metadata_http_honesty.py b/tests/test_chat_openai_metadata_http_honesty.py new file mode 100644 index 000000000..322d1072a --- /dev/null +++ b/tests/test_chat_openai_metadata_http_honesty.py @@ -0,0 +1,216 @@ +"""Chat/Completions OpenAI metadata shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_openai_metadata_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_string_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta ok"}], + "metadata": {"request_id": "req-1", "tenant": "acme"}, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta string"}], + "metadata": "not-an-object", + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_non_string_value() -> None: + """Buyers must not store untyped junk that observability would silently drop.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta int value"}], + "metadata": {"count": 3}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_metadata" in blob + assert "strings" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_too_many_entries() -> None: + server, thread, port = _server() + try: + meta = {f"k{i:02d}": f"v{i}" for i in range(17)} + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta overflow"}], + "metadata": meta, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + assert "16" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_key_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta key long"}], + "metadata": {"k" * 65: "v"}, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + assert "64" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_value_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta value long"}], + "metadata": {"k": "v" * 513}, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + assert "512" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_string_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy meta", + "metadata": {"source": "cli"}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_metadata_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no meta"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_string_metadata() + test_http_chat_rejects_metadata_non_object() + test_http_chat_rejects_metadata_non_string_value() + test_http_chat_rejects_metadata_too_many_entries() + test_http_chat_rejects_metadata_key_too_long() + test_http_chat_rejects_metadata_value_too_long() + test_http_completions_accepts_string_metadata() + test_http_chat_accepts_metadata_omitted() + print("ok") diff --git a/tests/test_chat_orchestration_mode_http_honesty.py b/tests/test_chat_orchestration_mode_http_honesty.py new file mode 100644 index 000000000..8ce21f8e6 --- /dev/null +++ b/tests/test_chat_orchestration_mode_http_honesty.py @@ -0,0 +1,136 @@ +"""Chat Completions mode/orchestration_mode: auto|route|conduct; invalid fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_orchestration_mode_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ + ModelAgent("planner_agent", "mock-planner", tags=("planning", "reasoning")), + ModelAgent("builder_agent", "mock-builder", tags=("coding", "writing")), + ModelAgent("reviewer_agent", "mock-reviewer", tags=("verification", "review")), + ] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def test_http_chat_accepts_mode_route() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "mode": "route", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_orchestration_mode_auto() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "orchestration_mode": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_invalid_mode() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "mode": "cascade", + }, + ) + assert status == 400, body + assert "invalid_mode" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_mode_non_string() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "orchestration": 1, + }, + ) + assert status == 400, body + assert "invalid_mode" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_mode_route() + test_http_chat_accepts_orchestration_mode_auto() + test_http_chat_rejects_invalid_mode() + test_http_chat_rejects_mode_non_string() + print("ok") diff --git a/tests/test_chat_parallel_tool_calls_http_honesty.py b/tests/test_chat_parallel_tool_calls_http_honesty.py new file mode 100644 index 000000000..8ad66c481 --- /dev/null +++ b/tests/test_chat_parallel_tool_calls_http_honesty.py @@ -0,0 +1,146 @@ +"""Chat parallel_tool_calls honesty: boolean type; true requires tools.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_parallel_tool_calls_honesty_token" # noqa: S105 + +_TOOLS = [ + { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up an invoice by id", + "parameters": { + "type": "object", + "properties": {"invoice_id": {"type": "string"}}, + "required": ["invoice_id"], + }, + }, + } +] + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_parallel_tool_calls_false_without_tools_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "parallel_tool_calls": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_parallel_tool_calls_true_without_tools_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "parallel_tool_calls": True, + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_parallel_tool_calls_non_boolean_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "parallel_tool_calls": "yes", + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_parallel_tool_calls_true_with_tools_passthrough() -> None: + """With tools, parallel_tool_calls triggers single-agent passthrough path.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "lookup invoice 42"}], + "tools": _TOOLS, + "parallel_tool_calls": True, + }, + ) + # Mock passthrough returns chat-shaped body + assert status == 200, body + assert "choices" in body or "id" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_parallel_tool_calls_false_without_tools_ok() + test_http_chat_parallel_tool_calls_true_without_tools_fail_closed() + test_http_chat_parallel_tool_calls_non_boolean_fail_closed() + test_http_chat_parallel_tool_calls_true_with_tools_passthrough() + print("ok") diff --git a/tests/test_chat_penalties_http_honesty.py b/tests/test_chat_penalties_http_honesty.py new file mode 100644 index 000000000..0a6b5d193 --- /dev/null +++ b/tests/test_chat_penalties_http_honesty.py @@ -0,0 +1,171 @@ +"""Chat/Completions presence_penalty and frequency_penalty honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_penalties_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_presence_and_frequency_penalty() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "penalties in range"}], + "presence_penalty": 0.5, + "frequency_penalty": -0.25, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_presence_penalty_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "presence high"}], + "presence_penalty": 2.5, + }, + ) + assert status == 400, body + assert "invalid_presence_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_frequency_penalty_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "frequency low"}], + "frequency_penalty": -3, + }, + ) + assert status == 400, body + assert "invalid_frequency_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_presence_penalty_non_number() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "presence bool"}], + "presence_penalty": True, + }, + ) + assert status == 400, body + assert "invalid_presence_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_penalties_in_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy penalties", + "presence_penalty": 1.0, + "frequency_penalty": 1.0, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_penalties_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no penalties"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_presence_and_frequency_penalty() + test_http_chat_rejects_presence_penalty_out_of_range() + test_http_chat_rejects_frequency_penalty_out_of_range() + test_http_chat_rejects_presence_penalty_non_number() + test_http_completions_accepts_penalties_in_range() + test_http_chat_accepts_penalties_omitted() + print("ok") diff --git a/tests/test_chat_prediction_http_honesty.py b/tests/test_chat_prediction_http_honesty.py new file mode 100644 index 000000000..687e89542 --- /dev/null +++ b/tests/test_chat_prediction_http_honesty.py @@ -0,0 +1,152 @@ +"""Chat Completions prediction (Predicted Outputs) honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_prediction_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_prediction_object() -> None: + """Buyers must not believe Predicted Outputs latency optimization was applied.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict me"}], + "prediction": { + "type": "content", + "content": "expected completion text", + }, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_prediction" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_prediction_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict string"}], + "prediction": "expected", + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_prediction_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict null"}], + "prediction": None, + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_prediction_bool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict bool"}], + "prediction": True, + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_prediction_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no prediction"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_prediction_object() + test_http_chat_rejects_prediction_string() + test_http_chat_rejects_prediction_null() + test_http_chat_rejects_prediction_bool() + test_http_chat_accepts_prediction_omitted() + print("ok") diff --git a/tests/test_chat_response_format_http_honesty.py b/tests/test_chat_response_format_http_honesty.py new file mode 100644 index 000000000..59b194a60 --- /dev/null +++ b/tests/test_chat_response_format_http_honesty.py @@ -0,0 +1,218 @@ +"""Chat Completions response_format honesty over HTTP (structured-output shape).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_response_format_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_response_format_text() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "plain text"}], + "response_format": {"type": "text"}, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_response_format_json_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "json object mode"}], + "response_format": {"type": "json_object"}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_valid_json_schema_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "structured"}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "receipt_line", + "schema": { + "type": "object", + "properties": {"amount": {"type": "number"}}, + }, + "strict": True, + }, + }, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_unknown_response_format_type() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad type"}], + "response_format": {"type": "xml"}, + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_json_object_with_sibling_keys() -> None: + """Buyers must not smuggle extra fields into type-only response_format objects.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "sibling"}], + "response_format": {"type": "json_object", "strict": True}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "only the type field" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_json_schema_without_schema_body() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "missing schema"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "receipt_line"}, + }, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "schema must be an object" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_non_object_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "string fmt"}], + "response_format": "json", + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_response_format_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no format"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_response_format_text() + test_http_chat_accepts_response_format_json_object() + test_http_chat_accepts_valid_json_schema_response_format() + test_http_chat_rejects_unknown_response_format_type() + test_http_chat_rejects_json_object_with_sibling_keys() + test_http_chat_rejects_json_schema_without_schema_body() + test_http_chat_rejects_non_object_response_format() + test_http_chat_accepts_response_format_omitted() + print("ok") diff --git a/tests/test_chat_service_tier_http_honesty.py b/tests/test_chat_service_tier_http_honesty.py new file mode 100644 index 000000000..7b7477341 --- /dev/null +++ b/tests/test_chat_service_tier_http_honesty.py @@ -0,0 +1,192 @@ +"""Chat Completions service_tier honesty over HTTP (capacity priority knob).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_service_tier_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_service_tier_auto() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "auto tier"}], + "service_tier": "auto", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_service_tier_default() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "default tier"}], + "service_tier": "default", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_service_tier_flex() -> None: + """flex/priority are capacity modes this gateway does not apply — fail closed.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "flex tier"}], + "service_tier": "flex", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_service_tier" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_service_tier_priority() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "priority tier"}], + "service_tier": "priority", + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_service_tier_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad tier type"}], + "service_tier": 1, + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_service_tier_flex() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy flex", + "service_tier": "flex", + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_service_tier_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no tier"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_service_tier_auto() + test_http_chat_accepts_service_tier_default() + test_http_chat_rejects_service_tier_flex() + test_http_chat_rejects_service_tier_priority() + test_http_chat_rejects_service_tier_non_string() + test_http_completions_rejects_service_tier_flex() + test_http_chat_accepts_service_tier_omitted() + print("ok") diff --git a/tests/test_chat_store_http_honesty.py b/tests/test_chat_store_http_honesty.py new file mode 100644 index 000000000..016a2c6f6 --- /dev/null +++ b/tests/test_chat_store_http_honesty.py @@ -0,0 +1,149 @@ +"""Chat Completions store honesty over HTTP (OpenAI store persistence knob).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_store_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_store_true() -> None: + """Buyers must not believe store=true persisted a completion when it cannot.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "remember this"}], + "store": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_store" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_store_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no store please"}], + "store": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_store_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "default store omit"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_store_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad store type"}], + "store": "yes", + }, + ) + assert status == 400, body + assert "invalid_store" in json.dumps(body) + assert "boolean" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_store_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null store"}], + "store": None, + }, + ) + assert status == 400, body + assert "invalid_store" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_store_true() + test_http_chat_accepts_store_false() + test_http_chat_accepts_store_omitted() + test_http_chat_rejects_store_non_boolean() + test_http_chat_rejects_store_null() + print("ok") diff --git a/tests/test_chat_stream_options_http_honesty.py b/tests/test_chat_stream_options_http_honesty.py new file mode 100644 index 000000000..c6d47c0f3 --- /dev/null +++ b/tests/test_chat_stream_options_http_honesty.py @@ -0,0 +1,158 @@ +"""Chat stream_options honesty: requires stream=true; include_usage true fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_stream_options_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict | str]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + try: + return response.status, json.loads(raw) + except json.JSONDecodeError: + return response.status, raw + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8") + try: + return exc.code, json.loads(raw) + except json.JSONDecodeError: + return exc.code, raw + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_stream_options_without_stream_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream_options": {"include_usage": False}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_stream_options_include_usage_true_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_stream_options_include_usage_false_with_stream_ok() -> None: + """stream=true with include_usage=false is accepted (usage not requested).""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": False}, + }, + ) + # Streaming may return 200 SSE body; accept 200 + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_stream_options_non_object_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": "include_usage", + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_omits_stream_options_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + }, + ) + assert status == 200, body + assert isinstance(body, dict) and "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_stream_options_without_stream_fail_closed() + test_http_chat_stream_options_include_usage_true_fail_closed() + test_http_chat_stream_options_include_usage_false_with_stream_ok() + test_http_chat_stream_options_non_object_fail_closed() + test_http_chat_omits_stream_options_ok() + print("ok") diff --git a/tests/test_chat_temperature_top_p_http_honesty.py b/tests/test_chat_temperature_top_p_http_honesty.py new file mode 100644 index 000000000..c080a7ed3 --- /dev/null +++ b/tests/test_chat_temperature_top_p_http_honesty.py @@ -0,0 +1,212 @@ +"""Chat/Completions temperature and top_p sampling honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_temperature_top_p_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_temperature_and_top_p_in_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "sample in range"}], + "temperature": 0.7, + "top_p": 0.9, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_temperature_above_two() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hot"}], + "temperature": 2.5, + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_temperature_negative() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "cold"}], + "temperature": -0.1, + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_temperature_bool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bool temp"}], + "temperature": True, + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_top_p_zero() -> None: + """top_p must be in (0, 1]; zero is not a valid nucleus mass.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "top_p zero"}], + "top_p": 0, + }, + ) + assert status == 400, body + assert "invalid_top_p" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_top_p_above_one() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "top_p high"}], + "top_p": 1.1, + }, + ) + assert status == 400, body + assert "invalid_top_p" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_temperature_bounds() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy temp bounds", + "temperature": 0, + "top_p": 1, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_sampling_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "defaults"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_temperature_and_top_p_in_range() + test_http_chat_rejects_temperature_above_two() + test_http_chat_rejects_temperature_negative() + test_http_chat_rejects_temperature_bool() + test_http_chat_rejects_top_p_zero() + test_http_chat_rejects_top_p_above_one() + test_http_completions_accepts_temperature_bounds() + test_http_chat_accepts_sampling_omitted() + print("ok") diff --git a/tests/test_chat_tool_call_id_http_honesty.py b/tests/test_chat_tool_call_id_http_honesty.py new file mode 100644 index 000000000..ab0978338 --- /dev/null +++ b/tests/test_chat_tool_call_id_http_honesty.py @@ -0,0 +1,170 @@ +"""Chat Completions tool message tool_call_id honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_tool_call_id_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_tool_message_with_tool_call_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + { + "role": "tool", + "content": "result payload", + "tool_call_id": "call_abc123", + }, + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_message_missing_tool_call_id() -> None: + """Buyers must not bind tool results without a tool_call_id.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + {"role": "tool", "content": "orphan result"}, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob + assert "tool_call_id" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_message_blank_tool_call_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + {"role": "tool", "content": "blank id", "tool_call_id": " "}, + ], + }, + ) + assert status == 400, body + assert "tool_call_id" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_call_id_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + { + "role": "tool", + "content": "long id", + "tool_call_id": "c" * 129, + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "tool_call_id" in blob + assert "128" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_call_id_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + {"role": "tool", "content": "num id", "tool_call_id": 42}, + ], + }, + ) + assert status == 400, body + assert "tool_call_id" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_tool_message_with_tool_call_id() + test_http_chat_rejects_tool_message_missing_tool_call_id() + test_http_chat_rejects_tool_message_blank_tool_call_id() + test_http_chat_rejects_tool_call_id_too_long() + test_http_chat_rejects_tool_call_id_non_string() + print("ok") diff --git a/tests/test_chat_tool_choice_functions_http_honesty.py b/tests/test_chat_tool_choice_functions_http_honesty.py new file mode 100644 index 000000000..6077fc1ca --- /dev/null +++ b/tests/test_chat_tool_choice_functions_http_honesty.py @@ -0,0 +1,154 @@ +"""Chat tools honesty: functions/function_call rejected; tool_choice requires tools.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_tool_choice_functions_http_honesty_token" # noqa: S105 + +_TOOLS = [ + { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up invoice by id", + "parameters": { + "type": "object", + "properties": {"invoice_id": {"type": "string"}}, + "required": ["invoice_id"], + }, + }, + } +] + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_functions_legacy_surface() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lookup invoice 9"}], + "functions": [ + { + "name": "lookup_invoice", + "description": "legacy", + "parameters": {"type": "object", "properties": {}}, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_functions" in blob + assert "tools" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_function_call_legacy_surface() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lookup invoice 9"}], + "function_call": "auto", + }, + ) + assert status == 400, body + assert "invalid_functions" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_choice_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi"}], + "tool_choice": "auto", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tool_choice" in blob + assert "tools" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_tools_with_tool_choice_passthrough_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lookup invoice 9"}], + "tools": _TOOLS, + "tool_choice": "auto", + }, + ) + assert status == 200, body + assert "choices" in body or "id" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_functions_legacy_surface() + test_http_chat_rejects_function_call_legacy_surface() + test_http_chat_rejects_tool_choice_without_tools() + test_http_chat_tools_with_tool_choice_passthrough_ok() + print("ok") diff --git a/tests/test_chat_unknown_fields_http_honesty.py b/tests/test_chat_unknown_fields_http_honesty.py new file mode 100644 index 000000000..4bde80bc6 --- /dev/null +++ b/tests/test_chat_unknown_fields_http_honesty.py @@ -0,0 +1,182 @@ +"""Chat/Completions unknown request fields honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_unknown_fields_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_unknown_request_field() -> None: + """Buyers must not believe unsupported OpenAI-adjacent knobs were applied.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "unknown knob"}], + "audio": {"voice": "alloy", "format": "mp3"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "unknown_fields" in blob + assert "audio" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_multiple_unknown_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "two unknowns"}], + "web_search_options": {"search_context_size": "medium"}, + "prompt_cache_key": "cache-1", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "unknown_fields" in blob + assert "web_search_options" in blob or "prompt_cache_key" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_unknown_request_field() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy unknown", + "audio": {"voice": "alloy"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "unknown_fields" in blob + assert "audio" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_stream_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stream string"}], + "stream": "yes", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "stream must be a boolean" in blob or "invalid_request" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_invalid_mode() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad mode"}], + "mode": "turbo", + }, + ) + assert status == 400, body + assert "invalid_mode" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_known_fields_only() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "known only"}], + "temperature": 0.5, + "mode": "route", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_unknown_request_field() + test_http_chat_rejects_multiple_unknown_fields() + test_http_completions_rejects_unknown_request_field() + test_http_chat_rejects_stream_non_boolean() + test_http_chat_rejects_invalid_mode() + test_http_chat_accepts_known_fields_only() + print("ok") diff --git a/tests/test_commercial_readiness.py b/tests/test_commercial_readiness.py index a07f3ebe7..b357fde9f 100644 --- a/tests/test_commercial_readiness.py +++ b/tests/test_commercial_readiness.py @@ -172,7 +172,7 @@ def test_commercial_readiness_endpoint_openapi_admin_and_docs_contract() -> None ) chat_status, _ = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "Analyze, verify, and summarize commercial readiness."}]}, + {"model": "mock-planner", "messages": [{"role": "user", "content": "Analyze, verify, and summarize commercial readiness."}]}, "inference_secret", ) readiness_status, readiness = get_json( diff --git a/tests/test_completions_legacy_knobs_http_honesty.py b/tests/test_completions_legacy_knobs_http_honesty.py new file mode 100644 index 000000000..6281bca7c --- /dev/null +++ b/tests/test_completions_legacy_knobs_http_honesty.py @@ -0,0 +1,204 @@ +"""Completions best_of/echo/suffix/logprobs honesty: HTTP fail-closed contracts.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_legacy_knobs_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_baseline_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "draft a one-line payment receipt"}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_best_of_one_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "best_of": 1}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_best_of_multi_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "best_of": 3}, + ) + assert status == 400, body + assert "invalid_best_of" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_echo_false_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "echo": False}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_echo_true_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "echo": True}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_echo" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_empty_suffix_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "suffix": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_nonempty_suffix_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "function head(", + "suffix": ") { return 1; }", + }, + ) + assert status == 400, body + assert "invalid_suffix" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_logprobs_false_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "logprobs": False}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_logprobs_integer_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "logprobs": 5}, + ) + assert status == 400, body + assert "invalid_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_logprobs_true_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "logprobs": True}, + ) + assert status == 400, body + assert "invalid_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_baseline_ok() + test_http_completions_best_of_one_ok() + test_http_completions_best_of_multi_fail_closed() + test_http_completions_echo_false_ok() + test_http_completions_echo_true_fail_closed() + test_http_completions_empty_suffix_ok() + test_http_completions_nonempty_suffix_fail_closed() + test_http_completions_logprobs_false_ok() + test_http_completions_logprobs_integer_fail_closed() + test_http_completions_logprobs_true_fail_closed() + print("ok") diff --git a/tests/test_completions_max_tokens_http_honesty.py b/tests/test_completions_max_tokens_http_honesty.py new file mode 100644 index 000000000..4f6e6870e --- /dev/null +++ b/tests/test_completions_max_tokens_http_honesty.py @@ -0,0 +1,135 @@ +"""Completions max_tokens is applied to the provider client for the request.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + SecurityConfig, + build_server, +) + +_TEST_AUTH_TOKEN = "cmpl_max_tokens_pass_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def test_http_max_tokens_applies_and_restores() -> None: + orch = build() + default_cap = orch.client.max_output_tokens + server = build_server(orch, port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": 64}, + ) + assert status == 200, body + assert body["object"] == "text_completion" + # Restored after request so later work uses the server default again. + assert orch.client.max_output_tokens == default_cap + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_rejects_non_positive_max_tokens() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": 0}, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_max_tokens" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_without_max_tokens_ok() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post(port, {"model": "mock-planner", "prompt": "hello"}) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_max_tokens_applies_and_restores() + test_http_rejects_non_positive_max_tokens() + test_http_without_max_tokens_ok() + + +def test_http_rejects_bool_max_tokens() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": True}, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_max_tokens" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_rejects_oversized_max_tokens() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": 2_000_000}, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_max_tokens" + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_prompt_shape_http_honesty.py b/tests/test_completions_prompt_shape_http_honesty.py new file mode 100644 index 000000000..57db6fab2 --- /dev/null +++ b/tests/test_completions_prompt_shape_http_honesty.py @@ -0,0 +1,118 @@ +"""Legacy Completions prompt shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_prompt_shape_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_string_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": "hello buyer"}) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_string_array_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": ["line one", "line two"]}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_empty_string_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": " "}) + assert status == 400, body + assert "invalid_prompt" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_empty_array_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": []}) + assert status == 400, body + assert "invalid_prompt" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_blank_array_item() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": ["ok", " "]}, + ) + assert status == 400, body + assert "invalid_prompt" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_missing_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner"}) + assert status == 400, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_seed_http_honesty.py b/tests/test_completions_seed_http_honesty.py new file mode 100644 index 000000000..093807ed1 --- /dev/null +++ b/tests/test_completions_seed_http_honesty.py @@ -0,0 +1,150 @@ +"""Chat Completions seed honesty over HTTP (deterministic sampling knob not applied).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_seed_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_seed_integer() -> None: + """Seed is type-checked then rejected: gateway does not apply deterministic sampling.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "deterministic please"}], + "seed": 42, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_seed" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_seed_zero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "seed zero"}], + "seed": 0, + }, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_seed_non_integer() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "seed float"}], + "seed": 1.5, + }, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_seed_bool() -> None: + """JSON true must not coerce to integer seed 1.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "seed bool"}], + "seed": True, + }, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_seed_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no seed"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_seed_integer() + test_http_chat_rejects_seed_zero() + test_http_chat_rejects_seed_non_integer() + test_http_chat_rejects_seed_bool() + test_http_chat_accepts_seed_omitted() + print("ok") diff --git a/tests/test_completions_stop_http_honesty.py b/tests/test_completions_stop_http_honesty.py new file mode 100644 index 000000000..b494f2348 --- /dev/null +++ b/tests/test_completions_stop_http_honesty.py @@ -0,0 +1,156 @@ +"""Chat Completions stop-sequence honesty over HTTP (not applied on gateway).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_stop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_stop_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop me"}], + "stop": "END", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stop" in blob + assert "not supported" in blob + assert "chat" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_stop_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop list"}], + "stop": ["END", "STOP"], + }, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_stop_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty stop"}], + "stop": "", + }, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_stop_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy stop", + "stop": "END", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stop" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_stop_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no stop"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_stop_string() + test_http_chat_rejects_stop_array() + test_http_chat_rejects_empty_stop_string() + test_http_completions_rejects_stop_string() + test_http_chat_accepts_stop_omitted() + print("ok") diff --git a/tests/test_completions_stream_options_http_honesty.py b/tests/test_completions_stream_options_http_honesty.py new file mode 100644 index 000000000..665fac739 --- /dev/null +++ b/tests/test_completions_stream_options_http_honesty.py @@ -0,0 +1,130 @@ +"""Completions stream_options: requires stream=true (which itself fails closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_stream_options_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_stream_options_without_stream_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hi", + "stream_options": {"include_usage": False}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stream_options" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_stream_true_with_stream_options_fail_closed() -> None: + """stream=true is unsupported on Completions; stream_options cannot enable it.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hi", + "stream": True, + "stream_options": {"include_usage": False}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + # either invalid_stream or invalid_stream_options depending on validation order + assert "invalid_stream" in blob or "invalid_stream_options" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_stream_options_non_object_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hi", + "stream": True, + "stream_options": "nope", + }, + ) + assert status == 400, body + assert "invalid_stream" in json.dumps(body) or "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_omits_stream_options_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi"}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_stream_options_without_stream_fail_closed() + test_http_completions_stream_true_with_stream_options_fail_closed() + test_http_completions_stream_options_non_object_fail_closed() + test_http_completions_omits_stream_options_ok() + print("ok") diff --git a/tests/test_completions_stream_reject_http_honesty.py b/tests/test_completions_stream_reject_http_honesty.py new file mode 100644 index 000000000..ccfd0673a --- /dev/null +++ b/tests/test_completions_stream_reject_http_honesty.py @@ -0,0 +1,101 @@ +"""Legacy Completions stream honesty over HTTP (gateway rejects Completions streaming).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_stream_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_stream_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "stream": False}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_stream_true() -> None: + """Buyers must use /v1/chat/completions for streaming — Completions stream is unsupported.""" + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "stream": True}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "stream" in blob.lower() or "not supported" in blob.lower() + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_non_bool_stream() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "stream": "yes"}, + ) + assert status == 400, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_omits_stream_ok() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": "hi"}) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_cost_review_server.py b/tests/test_cost_review_server.py index fd27c5bc9..065bb1fbe 100644 --- a/tests/test_cost_review_server.py +++ b/tests/test_cost_review_server.py @@ -66,7 +66,8 @@ def test_chat_completion_reports_real_usage_and_records_cost() -> None: base = f"http://127.0.0.1:{port}" try: status, body = _request("POST", f"{base}/v1/chat/completions", token, - {"messages": [{"role": "user", "content": "hello there world"}], + {"model": "mock-a", + "messages": [{"role": "user", "content": "hello there world"}], "attribution": {"team": "alpha", "company": "acme"}}) assert status == 200 assert body["usage"]["total_tokens"] > 0 @@ -86,7 +87,8 @@ def test_batch_routing_via_chat_completion_and_results_retrieval() -> None: base = f"http://127.0.0.1:{port}" try: status, submitted = _request("POST", f"{base}/v1/chat/completions", token, - {"messages": [{"role": "user", "content": "batch this"}], + {"model": "mock-a", + "messages": [{"role": "user", "content": "batch this"}], "routing": {"latency_tolerant": True}, "attribution": {"company": "acme"}}) assert status == 202 diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index d50342289..2379c22da 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -35,6 +35,7 @@ def _build() -> TaskOrchestrator: def test_proxy_completion_forwards_response_format_and_returns_full_shape() -> None: orch = _build() body = { + "model": "mock-planner", "messages": [{"role": "user", "content": "extract JSON"}], "response_format": {"type": "json_schema", "json_schema": {"name": "x", "schema": {}}}, "temperature": 0.1, @@ -56,7 +57,7 @@ def test_proxy_completion_forwards_tools() -> None: orch = _build() tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}] result = orch.proxy_completion( - {"messages": [{"role": "user", "content": "call a tool"}], "tools": tools} + {"model": "mock-planner", "messages": [{"role": "user", "content": "call a tool"}], "tools": tools} ) assert result["echo"]["tools"] == tools @@ -102,6 +103,7 @@ def test_http_chat_completions_accepts_response_format_and_passes_through() -> N status, body = _post( url, { + "model": "mock-planner", "messages": [{"role": "user", "content": "give me JSON"}], "response_format": {"type": "json_object"}, }, @@ -118,7 +120,7 @@ def test_http_responses_endpoint_passes_through() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/responses" try: - status, body = _post(url, {"input": "hello", "tools": []}, token) + status, body = _post(url, {"model": "mock-planner", "input": "hello"}, token) finally: server.shutdown() assert status == 200 @@ -129,7 +131,7 @@ def test_http_plain_prompt_still_uses_orchestration_path() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" try: - status, body = _post(url, {"messages": [{"role": "user", "content": "hi"}]}, token) + status, body = _post(url, {"model": "mock-planner", "messages": [{"role": "user", "content": "hi"}]}, token) finally: server.shutdown() assert status == 200 diff --git a/tests/test_openai_user_field_http_honesty.py b/tests/test_openai_user_field_http_honesty.py new file mode 100644 index 000000000..65d552f68 --- /dev/null +++ b/tests/test_openai_user_field_http_honesty.py @@ -0,0 +1,174 @@ +"""OpenAI user field honesty on Completions and chat: empty/null fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "openai_user_field_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "hi", "user": "buyer_account_9"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "hi", "user": ""}, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_null_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "hi", "user": None}, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi"}], + "user": "buyer_account_9", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi"}], + "user": " ", + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "text-embedding-test", + "input": "semantic unit for buyer search", + "user": "buyer_account_9", + }, + ) + assert status == 200, body + assert body.get("object") == "list" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "text-embedding-test", "input": "x", "user": ""}, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_user() + test_http_completions_rejects_empty_user() + test_http_completions_rejects_null_user() + test_http_chat_accepts_user() + test_http_chat_rejects_empty_user() + test_http_embeddings_accepts_user() + test_http_embeddings_rejects_empty_user() + print("ok") diff --git a/tests/test_responses_instructions_reasoning_http_honesty.py b/tests/test_responses_instructions_reasoning_http_honesty.py new file mode 100644 index 000000000..648aed30c --- /dev/null +++ b/tests/test_responses_instructions_reasoning_http_honesty.py @@ -0,0 +1,170 @@ +"""Responses API instructions and reasoning honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_instructions_reasoning_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_nonempty_instructions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": "Be concise and factual.", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_blank_instructions() -> None: + """Empty instructions must not look like a configured system prompt.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": " ", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_instructions" in blob + assert "non-empty" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_instructions_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": ["Be concise"], + }, + ) + assert status == 400, body + assert "invalid_instructions" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_instructions_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": "x" * 32_001, + }, + ) + assert status == 400, body + assert "invalid_instructions" in json.dumps(body) + assert "32000" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_reasoning_object() -> None: + """Buyers must not believe o-series reasoning controls were applied on passthrough.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "think carefully", + "reasoning": {"effort": "high"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_reasoning" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_instructions_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello responses", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_nonempty_instructions() + test_http_responses_rejects_blank_instructions() + test_http_responses_rejects_instructions_non_string() + test_http_responses_rejects_instructions_too_long() + test_http_responses_rejects_reasoning_object() + test_http_responses_accepts_instructions_omitted() + print("ok") diff --git a/tests/test_responses_metadata_http_honesty.py b/tests/test_responses_metadata_http_honesty.py new file mode 100644 index 000000000..c15eef3f5 --- /dev/null +++ b/tests/test_responses_metadata_http_honesty.py @@ -0,0 +1,133 @@ +"""Responses API metadata honesty over HTTP (OpenAI string-map shape).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_metadata_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_string_metadata_map() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": {"tenant_id": "buyer-9", "channel": "api"}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_object_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": "not-an-object", + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_string_metadata_value() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": {"count": 3}, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_too_many_metadata_entries() -> None: + server, thread, port = _server() + try: + meta = {f"k{i}": f"v{i}" for i in range(17)} + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": meta, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_omits_metadata_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "summarize ledger"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_sales_readiness.py b/tests/test_sales_readiness.py index d690d5573..0ede97689 100644 --- a/tests/test_sales_readiness.py +++ b/tests/test_sales_readiness.py @@ -160,7 +160,7 @@ def test_sales_readiness_endpoint_openapi_and_admin_surface() -> None: ) chat_status, _ = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "Analyze, verify, and summarize readiness."}]}, + {"model": "mock-planner", "messages": [{"role": "user", "content": "Analyze, verify, and summarize readiness."}]}, "inference_secret", ) readiness_status, readiness = get_json( diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index 67134ea6b..d7ccc5083 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -42,7 +42,7 @@ def test_http_api_requires_bearer_token_and_hides_trace_by_default() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: unauthorized_status, unauthorized_body = post_json(f"http://127.0.0.1:{port}/v1/chat/completions", payload) @@ -72,7 +72,7 @@ def test_admin_and_inference_tokens_are_separate() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: admin_for_chat_status, _ = post_json( @@ -100,7 +100,7 @@ def test_loopback_without_configured_token_is_rejected() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: status, body = post_json(f"http://127.0.0.1:{port}/v1/chat/completions", payload) @@ -121,7 +121,7 @@ def test_http_api_validates_mode_and_request_shape() -> None: try: status, body = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "owner", "content": "hello"}], "orchestration": "unsafe"}, + {"model": "mock-generalist", "messages": [{"role": "owner", "content": "hello"}], "orchestration": "unsafe"}, token="secret_token", ) finally: @@ -141,7 +141,7 @@ def test_http_api_rejects_unknown_request_fields() -> None: try: status, body = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "hello"}], "unexpected": True}, + {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}], "unexpected": True}, token="secret_token", ) finally: @@ -161,7 +161,7 @@ def test_rate_limit_returns_429_after_configured_budget() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: first_status, _ = post_json(f"http://127.0.0.1:{port}/v1/chat/completions", payload, token="secret_token") diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 789fb7084..f1dd81e06 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -89,7 +89,7 @@ def _serve() -> tuple[object, int, str]: def test_http_stream_true_returns_event_stream_and_reconstructs_answer() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" - payload = {"messages": [{"role": "user", "content": "stream please"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "stream please"}]} try: # Non-streaming reference answer. _, ref_ct, ref_body = _post(url, payload, token) @@ -118,7 +118,7 @@ def test_http_stream_false_is_unchanged_json() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" try: - status, content_type, body = _post(url, {"messages": [{"role": "user", "content": "hi"}], "stream": False}, token) + status, content_type, body = _post(url, {"model": "mock-generalist", "messages": [{"role": "user", "content": "hi"}], "stream": False}, token) finally: server.shutdown() assert status == 200 @@ -132,7 +132,7 @@ def test_http_stream_non_boolean_is_rejected() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" try: - status, _, body = _post(url, {"messages": [{"role": "user", "content": "hi"}], "stream": "yes"}, token) + status, _, body = _post(url, {"model": "mock-generalist", "messages": [{"role": "user", "content": "hi"}], "stream": "yes"}, token) finally: server.shutdown() assert status == 400 diff --git a/tests/test_true_streaming.py b/tests/test_true_streaming.py index ef451955b..7220def82 100644 --- a/tests/test_true_streaming.py +++ b/tests/test_true_streaming.py @@ -118,8 +118,8 @@ def post(payload: dict) -> tuple[str, str]: return response.headers.get("content-type", ""), response.read().decode("utf-8") try: - content_type, sse = post({"messages": [{"role": "user", "content": "stream this"}], "mode": "route", "stream": True}) - _, ref = post({"messages": [{"role": "user", "content": "stream this"}], "mode": "route"}) + content_type, sse = post({"model": "m-model", "messages": [{"role": "user", "content": "stream this"}], "mode": "route", "stream": True}) + _, ref = post({"model": "m-model", "messages": [{"role": "user", "content": "stream this"}], "mode": "route"}) finally: server.shutdown()