diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..22e77e6ca --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog + +All notable changes to Contextual Orchestrator are recorded here. +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Fixed + +- Pop JSON-null `tool.function` `description` / `parameters` / `strict` in + place before provider proxy so SDK optional defaults match omit + (`#638` unique tip). +- Pop JSON-null or blank `response_format.json_schema.description` and + JSON-null `strict`; fail closed on unknown inner keys and non-string + descriptions so structured-output passthrough cannot smuggle or forward + null optionals. +- Pop empty or whitespace-only `tool.function.description` before proxy + so SDK blank defaults match omit. +- Fail closed on `response_format.json_schema.name` that is longer than + 64 characters or outside `[a-zA-Z0-9_-]` so buyers get + `invalid_response_format` instead of an opaque provider rejection. + ASCII is required: `str.isalnum()` alone accepted Unicode letters and + digits (`café`, `名前`, Arabic-Indic digits). The request-body fuzz + target now exercises that validator. +- Fail closed on `tools[].function.name` outside ASCII + `[a-zA-Z0-9_-]{1,64}` so buyers get `invalid_tools` instead of an + opaque provider rejection. The same `str.isalnum()` Unicode leak that + `#669` closed on `json_schema.name` applied here. + +### Documentation + +- Compatibility honesty next-action notes in `docs/rest_api_design.md`. +- APA 7th citations for OpenAI Chat Completions, Structured Outputs, and + IETF JSON Schema 2020-12 in `docs/papers/README.md`. diff --git a/conductor/tracks.md b/conductor/tracks.md index 968c08ef8..1f0bec414 100644 --- a/conductor/tracks.md +++ b/conductor/tracks.md @@ -4,3 +4,4 @@ |---|---|---| | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD | | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n | +| 003-compatibility-honesty | active | Pop SDK-null optionals before proxy; fail-closed unknown structured-output keys and illegal `json_schema.name`. Landing unique tips: #638 tool.function null pop, #646 json_schema inner omit, #654/#669 name charset ASCII, next unique: ASCII-strict `tool.function.name` (`str.isalnum()` leaked Unicode). Official Responses `text.format` stays on #657/#666. Do not merge 140-file honesty stacks onto `main`. | diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index d3943c5be..75b5734e4 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -222,6 +222,8 @@ class UsageRecord: def as_dict(self) -> Dict[str, Any]: """Flatten the record (attribution inlined) for JSON + SQL storage.""" + # Execution identity is evidence of what ran — never a client-chosen tag. + # Account/service/team/group/company remain descriptive attribution. row = { "usage_record_id": self.usage_record_id, "created_at": self.created_at, @@ -583,12 +585,12 @@ def _seed_dimension_catalog(self) -> None: ph = self._placeholder() cur = self._conn.cursor() for order, (name, label, _column) in enumerate(ATTRIBUTION_DIMENSION_CATALOG): - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: only the DB-API placeholder char is interpolated; the value is bound. f"SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = {ph}", # nosec B608 - ph is a DB-API placeholder. (name,), ) if cur.fetchone() is None: - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: only DB-API placeholder chars are interpolated; values are bound. "INSERT INTO cost_attribution_dimensions " f"(dimension_name, dimension_label, dimension_order) VALUES ({ph}, {ph}, {ph})", # nosec B608 - ph is a DB-API placeholder. (name, label, order), @@ -602,7 +604,7 @@ def append(self, record: UsageRecord) -> None: placeholders = ", ".join(ph for _ in _USAGE_COLUMNS) columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute( + cur.execute( # nosemgrep -- sqlalchemy-execute-raw-query FP: columns are the fixed _USAGE_COLUMNS constant; values are bound. f"INSERT INTO llm_usage_records ({columns}) VALUES ({placeholders})", # nosec B608 - columns are fixed _USAGE_COLUMNS. tuple(row.get(column) for column in _USAGE_COLUMNS), ) @@ -622,7 +624,7 @@ def query(self, start: Optional[int] = None, end: Optional[int] = None) -> List[ where = f" WHERE {' AND '.join(clauses)}" if clauses else "" columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. + cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. # nosemgrep -- sqlalchemy-execute-raw-query FP: fixed columns and clause templates; all values are bound. return [dict(zip(_USAGE_COLUMNS, values)) for values in cur.fetchall()] @@ -681,14 +683,31 @@ def record_usage( ) -> UsageRecord: """Compute cost, build a :class:`UsageRecord`, persist it, and return it.""" if isinstance(attribution, dict) or attribution is None: - dims = AttributionDimensions.from_mapping(attribution) + # Strip caller-controlled execution identity before mapping so a + # client cannot spoof model/provider rollups (buyer-bill honesty). + if isinstance(attribution, dict): + cleaned = { + key: value + for key, value in attribution.items() + if key not in {"model_name", "provider", "upstream_api"} + } + else: + cleaned = None + dims = AttributionDimensions.from_mapping(cleaned) else: - dims = attribution - # Keep the model_name dimension aligned with the served model unless the - # caller pinned it explicitly, and default the provider dimension too. - if dims.model_name == UNATTRIBUTED and model: + dims = AttributionDimensions( + account=attribution.account, + service=attribution.service, + upstream_api=UNATTRIBUTED, + model_name=UNATTRIBUTED, + team=attribution.team, + group=attribution.group, + company=attribution.company, + ) + # Execution identity always wins — descriptive dimensions stay as-is. + if model: dims.model_name = model - if dims.upstream_api == UNATTRIBUTED and provider: + if provider: dims.upstream_api = provider cost_amount, currency = self.price_book.compute_cost( diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0097b722e..47b12dcb3 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -29,7 +29,8 @@ from .credentials import NotConfigured, get_credential -ChatMessage = dict[str, str] +# content is usually str; multimodal vision messages use OpenAI content-parts lists. +ChatMessage = dict[str, Any] class BudgetExceededError(RuntimeError): """Raised when an operator-configured spend budget is already exhausted.""" @@ -215,6 +216,10 @@ def __init__( ) -> None: self.timeout = timeout self.max_output_tokens = max_output_tokens + self.default_temperature = 0.2 + self.default_top_p: float | None = None + self.default_presence_penalty: float | None = None + self.default_frequency_penalty: float | None = None self.max_retries = max_retries self.retry_backoff = retry_backoff self.retry_backoff_cap = retry_backoff_cap @@ -230,7 +235,7 @@ def __init__( @staticmethod def _build_ssl_context(ca_bundle: str | None, verify_tls: bool) -> ssl.SSLContext: if not verify_tls: - return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. + return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. # nosemgrep -- unverified-ssl-context: intentional, default-secure (verify_tls defaults True) dev-only opt-out for self-signed endpoints. if ca_bundle: if not os.path.isfile(ca_bundle): raise ValueError(f"provider CA bundle does not exist: {ca_bundle}") @@ -246,9 +251,29 @@ def take_usage(self) -> dict[str, Any] | None: self._local.usage = None return usage - def chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: float = 0.2) -> str: - """Send messages to a mock or OpenAI-compatible chat endpoint with retries.""" + def chat( + self, + agent: ModelAgent, + messages: list[ChatMessage], + temperature: float | None = None, + top_p: float | None = None, + ) -> str: + """Send messages to a mock or OpenAI-compatible chat endpoint with retries. + + When ``temperature``/``top_p`` are omitted, ``default_temperature`` and + ``default_top_p`` are used so request-scoped Completions sampling can be + applied without threading kwargs through every orchestrator hop. + """ self._local.usage = None + # Expose the effective sampling knobs for request-path tests / diagnostics. + effective_temperature = self.default_temperature if temperature is None else temperature + effective_top_p = self.default_top_p if top_p is None else top_p + effective_presence = self.default_presence_penalty + effective_frequency = self.default_frequency_penalty + self._local.last_temperature = effective_temperature + self._local.last_top_p = effective_top_p + self._local.last_presence_penalty = effective_presence + self._local.last_frequency_penalty = effective_frequency if agent.base_url.startswith("mock://"): return self._mock(agent, messages) @@ -262,10 +287,16 @@ def chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: floa payload = { # pragma: no cover "model": agent.model, "messages": messages, - "temperature": temperature, + "temperature": effective_temperature, "stream": False, "max_tokens": self.max_output_tokens, } + if effective_top_p is not None: # pragma: no cover + payload["top_p"] = effective_top_p + if effective_presence is not None: # pragma: no cover + payload["presence_penalty"] = effective_presence + if effective_frequency is not None: # pragma: no cover + payload["frequency_penalty"] = effective_frequency return self._send_with_retry(agent, payload) def _send_with_retry(self, agent: ModelAgent, payload: dict[str, Any]) -> str: @@ -307,7 +338,7 @@ def _send(self, agent: ModelAgent, payload: dict[str, Any]) -> str: def _open_provider(self, request: urllib.request.Request) -> Any: """Open a provider request built from a validated provider URL.""" - return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. + return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. # nosemgrep -- dynamic-urllib-use: URL is built by _provider_url after scheme/host validation; egress to loopback/private/reserved is blocked. request, timeout=self.timeout, context=self._ssl_context, @@ -634,6 +665,9 @@ def _coerce_input_text(value: Any) -> str: if isinstance(item, str): parts.append(item) elif isinstance(item, dict): + # OpenAI content-parts: {"type": "text", "text": "..."} + if isinstance(item.get("text"), str): + parts.append(item["text"]) content = item.get("content") if isinstance(content, str): parts.append(content) @@ -644,6 +678,15 @@ def _coerce_input_text(value: Any) -> str: return " ".join(parts) +def _coerce_message_content_text(content: Any) -> str: + """Best-effort plain text from chat message content (string or content-parts).""" + if isinstance(content, str): + return content + if isinstance(content, list): + return _coerce_input_text(content) + return "" + + def load_agents(path: str) -> list[ModelAgent]: # pragma: no cover """Load model agent definitions from an agents JSON file.""" with open(path, encoding="utf-8") as handle: @@ -879,7 +922,14 @@ def _reload_state(self) -> None: # Orchestration-only body keys that must not be forwarded to the provider. _ORCHESTRATION_ONLY_KEYS = frozenset( - {"orchestration", "orchestration_mode", "mode", "include_orchestration_trace"} + { + "orchestration", + "orchestration_mode", + "mode", + "include_orchestration_trace", + "attribution", + "routing", + } ) def proxy_completion( @@ -897,7 +947,23 @@ def proxy_completion( text = self._latest_user_text(messages) else: text = _coerce_input_text(body.get("input")) - agent = self._select_agent(text, "worker") + requested_model = body.get("model") + # When the client names a model, resolve a pool agent that actually serves + # that model id. Silent rewrite to an unrelated agent.model is a commercial + # honesty failure for OpenAI SDKs (passthrough tools/Responses paths). + if isinstance(requested_model, str) and requested_model.strip(): + matched = [ + agent + for agent in self.agents + if not getattr(agent, "disabled", False) and agent.model == requested_model + ] + if not matched: + raise ValueError( + f"model {requested_model!r} is not available in the agent pool" + ) + agent = matched[0] + else: + agent = self._select_agent(text, "worker") upstream = { key: value for key, value in body.items() @@ -1600,7 +1666,13 @@ def _needs_workflow(self, text: str) -> bool: return hits >= self.policy.conduct_hint_threshold or len(text) > 700 def _latest_user_text(self, messages: list[ChatMessage]) -> str: - return next((m.get("content", "") for m in reversed(messages) if m.get("role") == "user"), "") # pragma: no cover + for message in reversed(messages): + if message.get("role") != "user": + continue + text = _coerce_message_content_text(message.get("content", "")) + if text: + return text + return "" # pragma: no cover def _model_judge_verification(self, task: str, fallback: dict[str, Any]) -> dict[str, Any]: """Ask a model to judge the verifier report (fixes term-matching false negatives). @@ -1695,6 +1767,52 @@ def list_agents(self, page_number: int = 1, page_size: int = 10) -> list[dict[st end = start + page_size return [self._agent_to_admin_payload(agent) for agent in self.agents[start:end]] + def list_openai_models(self) -> dict[str, Any]: + """Return an OpenAI-compatible ``/v1/models`` list from the agent pool. + + Buyers discover selectable model ids without admin-scope agent pool access. + Each enabled agent model appears once; gateway default + ``contextual-orchestrator`` is always first. + """ + created = 1_700_000_000 # stable epoch so list responses are deterministic + data: list[dict[str, Any]] = [ + { + "id": "contextual-orchestrator", + "object": "model", + "created": created, + "owned_by": "contextual-orchestrator", + } + ] + seen: set[str] = {"contextual-orchestrator"} + for agent in self.agents: + if agent.disabled: + continue + model_id = str(agent.model).strip() + if not model_id or model_id in seen: + continue + seen.add(model_id) + data.append( + { + "id": model_id, + "object": "model", + "created": created, + "owned_by": agent.provider_name + or self._infer_provider_name(agent.base_url) + or "agent_pool", + } + ) + return {"object": "list", "data": data} + + def get_openai_model(self, model_id: str) -> dict[str, Any]: + """Return one OpenAI model object or raise ``KeyError`` when unknown.""" + wanted = (model_id or "").strip() + if not wanted: + raise KeyError(model_id) + for item in self.list_openai_models()["data"]: + if item["id"] == wanted: + return item + raise KeyError(model_id) + def list_recent_runs(self, page_number: int = 1, page_size: int = 10) -> list[dict[str, Any]]: """Return a paginated list of recent workflow run records.""" if page_number < 1 or page_size < 1: # pragma: no cover @@ -8517,6 +8635,29 @@ def chat_completion_response( } +def text_completion_response( + result: dict[str, Any], + model: str = "contextual-orchestrator", + usage: dict[str, int] | None = None, +) -> dict[str, Any]: # pragma: no cover + """Wrap orchestration output as OpenAI legacy ``text_completion`` (``/v1/completions``).""" + return { + "id": f"cmpl-{int(time.time() * 1000)}", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "text": result["answer"], + "logprobs": None, + "finish_reason": "stop", + } + ], + "usage": usage or {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + _STREAM_CHUNK_SIZE = 32 diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index c58d4cb79..322fdcd56 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -22,6 +22,7 @@ TaskOrchestrator, chat_completion_chunks, chat_completion_response, + text_completion_response, redact_value, sse_stream_body, ) @@ -32,21 +33,77 @@ "seed", "presence_penalty", "frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "user", "metadata", "parallel_tool_calls", "reasoning_effort", "response_format", "tools", "tool_choice", "functions", "function_call", - "modalities", "prediction", "store", "service_tier", + "modalities", "prediction", "store", "service_tier", "stream_options", + # Chat-era surfaces accepted only for explicit unsupported errors. + "audio", "web_search_options", + # Modern OpenAI SDK control fields — accepted only for named unsupported errors. + "prompt_cache_key", "safety_identifier", "verbosity", "prompt_cache_retention", + # Responses-style reasoning object on chat — named unsupported (not effort string). + "reasoning", + # Async background mode — not supported on this gateway. + "background", + "include", } # Provider features the multi-agent verifier cannot merge -> single-agent passthrough. PASSTHROUGH_TRIGGER_KEYS = {"response_format", "tools", "tool_choice", "functions", "function_call"} ALLOWED_CHAT_KEYS = { "model", "messages", "orchestration", "orchestration_mode", "mode", "include_orchestration_trace", "stream", "attribution", "routing", + # Tool-loop budget — accepted only for named unsupported error (no multi-step tool loop). + "max_tool_calls", } | OPENAI_PASSTHROUGH_PARAM_KEYS # Responses API body keys (`input` replaces `messages`). ALLOWED_RESPONSES_KEYS = { "model", "input", "instructions", "stream", "metadata", "reasoning", + # OpenAI Responses native output budget (not max_tokens on this surface). + "max_output_tokens", + # Tool-loop budget — accepted only for explicit unsupported error (no multi-step tool loop). + "max_tool_calls", + # Gateway cost/routing control plane (stripped before provider passthrough). + "attribution", "routing", + # OpenAI conversation-control surfaces — accepted only to fail closed with + # explicit unsupported errors (not opaque unknown_fields). + "previous_response_id", "conversation", "truncation", "include", "text", } | OPENAI_PASSTHROUGH_PARAM_KEYS ALLOWED_BATCH_KEYS = {"requests", "attribution", "routing", "model"} -ALLOWED_EMBEDDINGS_BATCH_KEYS = {"model", "input", "inputs", "endpoint", "metadata", "attribution"} +ALLOWED_EMBEDDINGS_BATCH_KEYS = {"model", "input", "inputs", "endpoint", "metadata", "attribution", "user", "encoding_format", "dimensions", "routing"} +ALLOWED_EMBEDDINGS_KEYS = { + "model", "input", "encoding_format", "dimensions", "user", "metadata", "attribution", "routing", +} +ALLOWED_COMPLETIONS_KEYS = { + "model", "prompt", "stream", "stream_options", "echo", "suffix", "best_of", + "logprobs", "top_logprobs", "n", "max_tokens", "max_completion_tokens", "temperature", "top_p", "stop", "user", "seed", + "presence_penalty", "frequency_penalty", "logit_bias", "service_tier", "metadata", + "store", + # Chat-era tool surfaces — accepted only for explicit unsupported errors. + "tools", "tool_choice", "functions", "function_call", "parallel_tool_calls", + # Tool-loop budget (chat/Responses-native) — named unsupported, not unknown_fields. + "max_tool_calls", + "response_format", + # Chat-era structured/output controls — accepted only for explicit migration errors. + "modalities", "prediction", "reasoning_effort", + # Chat-era multimodal/search — accepted only for named unsupported errors. + "audio", "web_search_options", + # Modern OpenAI SDK control fields — named unsupported errors. + "prompt_cache_key", "safety_identifier", "verbosity", "prompt_cache_retention", + "reasoning", "background", "include", +} | {"attribution", "routing"} ALLOWED_MESSAGE_ROLES = {"system", "user", "assistant", "tool"} +# Chat message object keys this gateway interprets. Anything else fails closed +# with unknown_message_fields (named error, not silent strip/smuggle). +ALLOWED_MESSAGE_KEYS = { + "role", + "content", + "name", + "tool_call_id", + "tool_calls", + "refusal", + "annotations", + "audio", + "function_call", + "weight", + "prefix", +} ALLOWED_MODES = {"auto", "route", "conduct"} ALLOWED_SIMULATE_KEYS = {"prompt", "mode", "include_orchestration_trace"} ALLOWED_WORKFLOW_KEYS = {"prompt_text", "run_mode", "include_orchestration_trace"} @@ -171,30 +228,1559 @@ def _coerce_json(payload: bytes) -> dict[str, Any]: return value + +def _validate_completion_prompt(prompt: Any) -> list[dict[str, str]]: + """Legacy Completions ``prompt`` → single user message list. + + Accepts a non-empty string or an array of strings (at most 128 items). OpenAI + also allows arrays of token IDs (integers); this gateway rejects token-id + prompts fail-closed with ``invalid_prompt`` so SDKs get a clear migration + path to string prompts. + """ + if isinstance(prompt, str): + if not prompt.strip(): + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + if len(prompt) > 32_000: + raise RequestError(400, "invalid_prompt", "prompt must be at most 32000 characters") + return [{"role": "user", "content": prompt}] + if isinstance(prompt, list): + if not prompt: + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + if len(prompt) > 128: + raise RequestError( + 400, + "invalid_prompt", + "prompt array must contain at most 128 items", + ) + # Token-id form: list of ints, or list of list of ints (batch of token sequences). + if all(isinstance(item, int) and not isinstance(item, bool) for item in prompt): + raise RequestError( + 400, + "invalid_prompt", + "token-id prompts are not supported; pass a string or array of strings", + ) + if all(isinstance(item, list) for item in prompt): + raise RequestError( + 400, + "invalid_prompt", + "token-id prompts are not supported; pass a string or array of strings", + ) + parts: list[str] = [] + for item in prompt: + if not isinstance(item, str): + raise RequestError(400, "invalid_prompt", "prompt array items must be strings") + if not item.strip(): + raise RequestError( + 400, + "invalid_prompt", + "prompt array items must be non-empty strings", + ) + parts.append(item) + joined = "\n".join(parts) + if not joined.strip(): + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + if len(joined) > 32_000: + raise RequestError(400, "invalid_prompt", "prompt must be at most 32000 characters") + return [{"role": "user", "content": joined}] + raise RequestError(400, "invalid_prompt", "prompt must be a non-empty string or array") + + +def _validate_completions_stream(body: dict[str, Any]) -> bool | None: + """Legacy Completions ``stream`` — strict boolean honesty contract. + + OpenAI Completions accepts streaming. This gateway: + - accepts omit and ``stream=false`` as the non-streaming text_completion path + - rejects ``stream=true`` with a clear redirect to chat completions + - rejects non-boolean values fail-closed (no silent coercion) + """ + if "stream" not in body: + return None + stream = body.get("stream") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if stream is None or (isinstance(stream, str) and not stream.strip()): + return None + if not isinstance(stream, bool): + raise RequestError(400, "invalid_stream", "stream must be a boolean") + if stream is True: + raise RequestError( + 400, + "invalid_stream", + "stream is not supported on /v1/completions; use /v1/chat/completions", + ) + return stream + + +def _validate_completions_echo(body: dict[str, Any]) -> bool | None: + """Legacy Completions ``echo`` — strict boolean; ``true`` is not supported. + + OpenAI can prepend the prompt to the completion when ``echo`` is true. This + gateway does not implement that behaviour, so ``echo=true`` fails closed with + a clear ``invalid_echo`` error. ``false`` and omit remain valid. + """ + if "echo" not in body: + return None + echo = body.get("echo") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if echo is None or (isinstance(echo, str) and not echo.strip()): + return None + if not isinstance(echo, bool): + raise RequestError(400, "invalid_echo", "echo must be a boolean") + if echo is True: + raise RequestError( + 400, + "invalid_echo", + "echo=true is not supported on /v1/completions", + ) + return echo + + + + + + + + + + + + + + +def _validate_completions_logit_bias(body: dict[str, Any]) -> dict[str, float] | None: + """Legacy Completions ``logit_bias`` — empty object is a no-op; non-empty fails closed. + + OpenAI uses logit_bias to bias token sampling. This gateway does not apply + token biases on the Completions route. An empty object is an honest no-op + (SDK clients often send ``{}``). Any non-empty map is type-checked then + rejected so clients never believe sampling bias was applied. + """ + if "logit_bias" not in body: + return None + bias = body.get("logit_bias") + # Explicit JSON null is treat-as-omit (SDK optional default). + if bias is None: + return None + if not isinstance(bias, dict): + raise RequestError(400, "invalid_logit_bias", "logit_bias must be an object of token biases") + # Empty object: no tokens to bias — treat as omit (honest no-op). + if len(bias) == 0: + return {} + if len(bias) > 300: + raise RequestError(400, "invalid_logit_bias", "logit_bias must contain at most 300 entries") + for key, value in bias.items(): + token = str(key) + if not token.isdigit(): + raise RequestError(400, "invalid_logit_bias", "logit_bias keys must be digit token ids") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RequestError(400, "invalid_logit_bias", "logit_bias values must be numbers in [-100, 100]") + number = float(value) + if number < -100 or number > 100: + raise RequestError(400, "invalid_logit_bias", "logit_bias values must be numbers in [-100, 100]") + raise RequestError( + 400, + "invalid_logit_bias", + "logit_bias is not supported on /v1/completions", + ) + + + +def _validate_service_tier(body: dict[str, Any], *, endpoint_path: str) -> str | None: + """OpenAI ``service_tier`` — accept omit/auto/default as no-ops; reject others. + + OpenAI uses service_tier for capacity priority (auto/default/flex/priority). + This gateway has no tiered capacity plane, so only auto/default (or omit/null) + are honest no-ops. Other values fail closed so clients cannot silently + believe flex/priority processing was applied. + """ + if "service_tier" not in body: + return None + service_tier = body.get("service_tier") + # Explicit JSON null or empty string is treat-as-omit (SDK optional default). + if service_tier is None or (isinstance(service_tier, str) and not service_tier.strip()): + return None + if not isinstance(service_tier, str): + raise RequestError(400, "invalid_service_tier", "service_tier must be a string") + # Strip incidental whitespace so " auto " matches auto (honest no-op). + service_tier = service_tier.strip() + if service_tier not in ("auto", "default"): + raise RequestError( + 400, + "invalid_service_tier", + f"service_tier values other than auto or default are not supported on {endpoint_path}", + ) + return service_tier + + +def _validate_completions_user(body: dict[str, Any]) -> str | None: + """OpenAI ``user`` end-user id — optional string, max 64 characters. + + Explicit JSON null is treat-as-omit (SDK optional default). Empty or + whitespace-only strings still fail closed so clients cannot attribute spend + to a blank identity. + """ + if "user" not in body: + return None + user = body.get("user") + # Explicit JSON null is treat-as-omit (SDK optional default). + if user is None: + return None + if not isinstance(user, str): + raise RequestError(400, "invalid_user", "user must be a string of at most 64 characters") + if not user.strip(): + raise RequestError(400, "invalid_user", "user must be a non-empty string of at most 64 characters") + if len(user) > 64: + raise RequestError(400, "invalid_user", "user must be a string of at most 64 characters") + return user + +def _validate_completions_n(body: dict[str, Any]) -> int | None: + """Legacy Completions ``n`` — positive integer; only ``n=1`` is supported. + + OpenAI can return multiple completions when ``n > 1``. This gateway always + returns a single choice, so ``n > 1`` fails closed. ``n=1`` and omit remain + valid. Cap 128 is retained for clear range errors before the support check. + """ + if "n" not in body: + return None + n = body.get("n") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if n is None or (isinstance(n, str) and not n.strip()): + return None + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise RequestError(400, "invalid_n", "n must be a positive integer") + if n > 128: + raise RequestError(400, "invalid_n", "n must be at most 128") + if n > 1: + raise RequestError( + 400, + "invalid_n", + "n greater than 1 is not supported on /v1/completions", + ) + return n + + +def _validate_responses_n(body: dict[str, Any]) -> int | None: + """Responses ``n`` — only omit or 1; multi-choice is not framed on passthrough. + + OpenAI may request multiple samples via ``n``. This gateway's Responses + passthrough returns a single completion shape, so ``n`` greater than 1 + fails closed. ``n=1`` and omit remain valid. + """ + if "n" not in body: + return None + n = body.get("n") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if n is None or (isinstance(n, str) and not n.strip()): + return None + if isinstance(n, bool) or not isinstance(n, int): + raise RequestError(400, "invalid_n", "n must be an integer") + if n < 1: + raise RequestError(400, "invalid_n", "n must be a positive integer") + if n > 1: + raise RequestError( + 400, + "invalid_n", + "n greater than 1 is not supported on /v1/responses", + ) + return n + + + +def _validate_responses_logit_bias(body: dict[str, Any]) -> dict[str, float] | None: + """Responses ``logit_bias`` — digit-token map values in [-100, 100]; pass through. + + Invalid shapes fail closed before provider egress. Valid maps (including empty) + are forwarded on Responses passthrough. + """ + if "logit_bias" not in body: + return None + bias = body.get("logit_bias") + # Explicit JSON null is treat-as-omit (SDK optional default). + if bias is None: + return None + if not isinstance(bias, dict): + raise RequestError(400, "invalid_logit_bias", "logit_bias must be an object of token biases") + if len(bias) > 300: + raise RequestError(400, "invalid_logit_bias", "logit_bias must contain at most 300 entries") + cleaned: dict[str, float] = {} + for key, value in bias.items(): + token = str(key) + if not token.isdigit(): + raise RequestError(400, "invalid_logit_bias", "logit_bias keys must be digit token ids") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RequestError(400, "invalid_logit_bias", "logit_bias values must be numbers in [-100, 100]") + number = float(value) + if number < -100 or number > 100: + raise RequestError(400, "invalid_logit_bias", "logit_bias values must be numbers in [-100, 100]") + cleaned[token] = number + return cleaned + + +def _validate_responses_logprobs(body: dict[str, Any]) -> None: + """Responses ``logprobs`` / ``top_logprobs`` — OpenAI shape; invalid fail closed. + + ``logprobs`` must be boolean when present. ``top_logprobs`` requires + ``logprobs=true`` and must be an integer in [0, 20]. + Explicit JSON null for either field is treat-as-omit (SDK optional default). + """ + if "logprobs" in body: + lp = body.get("logprobs") + if lp is not None and not isinstance(lp, bool): + raise RequestError(400, "invalid_logprobs", "logprobs must be a boolean") + if "top_logprobs" in body: + tlp = body.get("top_logprobs") + if tlp is None: + return + if body.get("logprobs") is not True: + raise RequestError( + 400, + "invalid_top_logprobs", + "top_logprobs requires logprobs=true on /v1/responses", + ) + if isinstance(tlp, bool) or not isinstance(tlp, int): + raise RequestError(400, "invalid_top_logprobs", "top_logprobs must be an integer in [0, 20]") + if tlp < 0 or tlp > 20: + raise RequestError(400, "invalid_top_logprobs", "top_logprobs must be an integer in [0, 20]") + + + +def _validate_responses_parallel_tool_calls(body: dict[str, Any]) -> bool | None: + """Responses ``parallel_tool_calls`` — strict boolean when present. + + OpenAI uses this flag to allow concurrent tool invocations. Invalid types + fail closed before provider passthrough so clients never believe a coerced + value was applied. ``true`` requires a non-empty ``tools`` array (chat parity). + """ + if "parallel_tool_calls" not in body: + return None + value = body.get("parallel_tool_calls") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if value is None or (isinstance(value, str) and not value.strip()): + return None + if not isinstance(value, bool): + raise RequestError( + 400, + "invalid_parallel_tool_calls", + "parallel_tool_calls must be a boolean", + ) + if value is True: + tools = body.get("tools") if "tools" in body else None + if not isinstance(tools, list) or not tools: + raise RequestError( + 400, + "invalid_parallel_tool_calls", + "parallel_tool_calls=true requires tools on /v1/responses", + ) + return value + + +def _validate_responses_seed(body: dict[str, Any]) -> int | None: + """Responses ``seed`` — signed int64; valid values pass through to the provider. + + Unlike Completions (where seed is not applied), Responses passthrough forwards + seed to the selected agent. Invalid types/ranges fail closed before egress. + """ + if "seed" not in body: + return None + seed = body.get("seed") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if seed is None or (isinstance(seed, str) and not seed.strip()): + return None + if isinstance(seed, bool) or not isinstance(seed, int): + raise RequestError(400, "invalid_seed", "seed must be an integer") + if seed < -(2**63) or seed > (2**63 - 1): + raise RequestError(400, "invalid_seed", "seed must fit in a signed 64-bit integer") + return seed + + +def _validate_responses_stop(body: dict[str, Any]) -> str | list[str] | None: + """Responses ``stop`` — string or ≤4 non-empty strings (≤256 chars); pass through. + + Shape matches OpenAI. Valid stop values are forwarded on Responses passthrough; + invalid shapes fail closed so clients never believe a broken stop list was applied. + """ + if "stop" not in body: + return None + stop = body.get("stop") + # Explicit JSON null is treat-as-omit (SDK optional default). + if stop is None: + return None + if isinstance(stop, str): + # Empty/whitespace string is omit-equivalent (no stop sequences). + if not stop.strip(): + return None + if len(stop) > 256: + raise RequestError(400, "invalid_stop", "each stop sequence must be at most 256 characters") + return stop + if isinstance(stop, list): + # Drop whitespace-only items; empty result is omit-equivalent. + stop = [item for item in stop if not (isinstance(item, str) and not item.strip())] + if not stop: + return None + if len(stop) > 4: + raise RequestError(400, "invalid_stop", "stop must be a string or array of up to 4 non-empty strings") + for item in stop: + if not isinstance(item, str) or not item: + raise RequestError(400, "invalid_stop", "stop sequences must be non-empty strings") + if len(item) > 256: + raise RequestError(400, "invalid_stop", "each stop sequence must be at most 256 characters") + return stop + raise RequestError(400, "invalid_stop", "stop must be a string or array of up to 4 non-empty strings") + + +def _validate_completions_stop(body: dict[str, Any]) -> str | list[str] | None: + """Legacy Completions ``stop`` — type-checked then rejected (not applied). + + OpenAI uses stop sequences to cut generation early. This gateway validates + shape (string or ≤4 non-empty strings, each ≤256 chars) but does not apply + stop sequences on the Completions path, so any provided non-empty ``stop`` fails closed. Empty string/array/null are omit no-ops. + """ + if "stop" not in body: + return None + stop = body.get("stop") + # Explicit JSON null is treat-as-omit (SDK optional default). + if stop is None: + return None + if isinstance(stop, str): + # Empty/whitespace string is omit-equivalent (no stop sequences). + if not stop.strip(): + return None + if len(stop) > 256: + raise RequestError(400, "invalid_stop", "each stop sequence must be at most 256 characters") + elif isinstance(stop, list): + # Drop whitespace-only items; empty result is omit-equivalent. + stop = [item for item in stop if not (isinstance(item, str) and not item.strip())] + if not stop: + return None + if len(stop) > 4: + raise RequestError(400, "invalid_stop", "stop must be a string or array of up to 4 non-empty strings") + for item in stop: + if not isinstance(item, str) or not item: + raise RequestError(400, "invalid_stop", "stop sequences must be non-empty strings") + if len(item) > 256: + raise RequestError(400, "invalid_stop", "each stop sequence must be at most 256 characters") + else: + raise RequestError(400, "invalid_stop", "stop must be a string or array of up to 4 non-empty strings") + raise RequestError( + 400, + "invalid_stop", + "stop sequences are not supported on /v1/completions", + ) + + + +def _validate_completions_seed(body: dict[str, Any]) -> int | None: + """Legacy Completions ``seed`` — type-checked then rejected (not applied). + + OpenAI uses seed for best-effort deterministic sampling. This gateway validates + signed int64 integers but does not apply seed on the Completions route path, + so any provided ``seed`` fails closed. Omit remains valid. + """ + if "seed" not in body: + return None + seed = body.get("seed") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if seed is None or (isinstance(seed, str) and not seed.strip()): + return None + if isinstance(seed, bool) or not isinstance(seed, int): + raise RequestError(400, "invalid_seed", "seed must be an integer") + if seed < -(2**63) or seed > (2**63 - 1): + raise RequestError(400, "invalid_seed", "seed must fit in a signed 64-bit integer") + raise RequestError( + 400, + "invalid_seed", + "seed is not supported on /v1/completions", + ) + + + +def _validate_completions_frequency_penalty(body: dict[str, Any]) -> float | None: + """Legacy Completions ``frequency_penalty`` — number in [-2, 2].""" + if "frequency_penalty" not in body: + return None + value = body.get("frequency_penalty") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if value is None or (isinstance(value, str) and not value.strip()): + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RequestError(400, "invalid_frequency_penalty", "frequency_penalty must be a number in [-2, 2]") + number = float(value) + if number < -2 or number > 2: + raise RequestError(400, "invalid_frequency_penalty", "frequency_penalty must be a number in [-2, 2]") + return number + +def _validate_completions_presence_penalty(body: dict[str, Any]) -> float | None: + """Legacy Completions ``presence_penalty`` — number in [-2, 2].""" + if "presence_penalty" not in body: + return None + value = body.get("presence_penalty") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if value is None or (isinstance(value, str) and not value.strip()): + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise RequestError(400, "invalid_presence_penalty", "presence_penalty must be a number in [-2, 2]") + number = float(value) + if number < -2 or number > 2: + raise RequestError(400, "invalid_presence_penalty", "presence_penalty must be a number in [-2, 2]") + return number + +def _validate_completions_temperature(body: dict[str, Any]) -> float | None: + """Legacy Completions ``temperature`` — number in [0, 2].""" + if "temperature" not in body: + return None + temperature = body.get("temperature") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if temperature is None or (isinstance(temperature, str) and not temperature.strip()): + return None + if isinstance(temperature, bool) or not isinstance(temperature, (int, float)): + raise RequestError(400, "invalid_temperature", "temperature must be a number in [0, 2]") + value = float(temperature) + if value < 0 or value > 2: + raise RequestError(400, "invalid_temperature", "temperature must be a number in [0, 2]") + return value + +def _validate_completions_top_p(body: dict[str, Any]) -> float | None: + """Legacy Completions ``top_p`` — number in (0, 1] (OpenAI nucleus sampling).""" + if "top_p" not in body: + return None + top_p = body.get("top_p") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if top_p is None or (isinstance(top_p, str) and not top_p.strip()): + return None + if isinstance(top_p, bool) or not isinstance(top_p, (int, float)): + raise RequestError(400, "invalid_top_p", "top_p must be a number in (0, 1]") + value = float(top_p) + if value <= 0 or value > 1: + raise RequestError(400, "invalid_top_p", "top_p must be a number in (0, 1]") + return value + +def _validate_completions_model(body: dict[str, Any]) -> str: + """Legacy Completions ``model`` — required non-empty string (OpenAI parity).""" + if "model" not in body: + raise RequestError(400, "invalid_model", "model is required") + model = body.get("model") + if not isinstance(model, str) or not model.strip(): + raise RequestError(400, "invalid_model", "model must be a non-empty string") + model = model.strip() + if len(model) > 256: + raise RequestError(400, "invalid_model", "model must be at most 256 characters") + return model + +def _validate_completions_max_tokens(body: dict[str, Any]) -> int | None: + """Legacy Completions ``max_tokens`` — positive integer capped at 1_048_576.""" + if "max_tokens" not in body: + return None + max_tokens = body.get("max_tokens") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if max_tokens is None or (isinstance(max_tokens, str) and not max_tokens.strip()): + return None + if isinstance(max_tokens, bool) or not isinstance(max_tokens, int) or max_tokens < 1: + raise RequestError(400, "invalid_max_tokens", "max_tokens must be a positive integer") + if max_tokens > 1_048_576: + raise RequestError( + 400, + "invalid_max_tokens", + "max_tokens must be at most 1048576", + ) + return max_tokens + +def _validate_chat_max_completion_tokens(body: dict[str, Any]) -> int | None: + """Chat Completions ``max_completion_tokens`` — positive integer capped at 1_048_576. + + OpenAI prefers this over legacy ``max_tokens`` for chat. When both are set, + ``max_completion_tokens`` wins so clients get a single honest budget. + """ + if "max_completion_tokens" not in body: + return None + max_completion_tokens = body.get("max_completion_tokens") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if max_completion_tokens is None or ( + isinstance(max_completion_tokens, str) and not max_completion_tokens.strip() + ): + return None + if ( + isinstance(max_completion_tokens, bool) + or not isinstance(max_completion_tokens, int) + or max_completion_tokens < 1 + ): + raise RequestError( + 400, + "invalid_max_completion_tokens", + "max_completion_tokens must be a positive integer", + ) + if max_completion_tokens > 1_048_576: + raise RequestError( + 400, + "invalid_max_completion_tokens", + "max_completion_tokens must be at most 1048576", + ) + return max_completion_tokens + + +def _validate_responses_max_output_tokens(body: dict[str, Any]) -> int | None: + """Responses ``max_output_tokens`` — OpenAI-native output budget (positive int). + + Official Responses clients send ``max_output_tokens`` rather than chat-era + ``max_tokens``. Accept and type-check so the field is not opaque + ``unknown_fields``; value is left on the body for provider passthrough. + Cap matches ``max_tokens`` (1_048_576). + """ + if "max_output_tokens" not in body: + return None + value = body.get("max_output_tokens") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if value is None or (isinstance(value, str) and not value.strip()): + return None + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise RequestError( + 400, + "invalid_max_output_tokens", + "max_output_tokens must be a positive integer", + ) + if value > 1_048_576: + raise RequestError( + 400, + "invalid_max_output_tokens", + "max_output_tokens must be at most 1048576", + ) + return value + + + +def _validate_max_tool_calls( + body: dict[str, Any], + *, + endpoint_path: str, +) -> None: + """Reject ``max_tool_calls`` — no multi-step tool loop on this gateway. + + OpenAI may cap tool-call rounds via ``max_tool_calls`` (Responses-native; + some chat SDKs also send it). This gateway proxies a single completion and + does not run a tool loop, so any provided value fails closed with a named + error rather than opaque ``unknown_fields``. Explicit JSON null and empty + / whitespace strings are treat-as-omit (SDK optional defaults). + """ + if "max_tool_calls" not in body: + return + value = body.get("max_tool_calls") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if value is None or (isinstance(value, str) and not value.strip()): + return + raise RequestError( + 400, + "invalid_max_tool_calls", + f"max_tool_calls is not supported on {endpoint_path}", + ) + + +def _validate_responses_max_tool_calls(body: dict[str, Any]) -> None: + """Responses ``max_tool_calls`` — named reject; null/empty omit.""" + _validate_max_tool_calls(body, endpoint_path="/v1/responses") + + +def _validate_completions_logprobs(body: dict[str, Any]) -> int | bool | None: + """Legacy Completions ``logprobs`` — only ``false``/omit; token logprobs unsupported. + + OpenAI accepts ``false`` or an integer 0–5 for top logprob counts. This gateway + always returns ``logprobs: null`` on text completions, so integer logprobs + (including 0–5) and boolean ``true`` fail closed. ``false`` and omit remain valid. + """ + if "logprobs" not in body: + return None + logprobs = body.get("logprobs") + # Explicit JSON null is treat-as-omit (SDK optional default). + if logprobs is None: + return None + if logprobs is False: + return False + if isinstance(logprobs, bool): # True + raise RequestError( + 400, + "invalid_logprobs", + "logprobs must be false; token logprobs are not supported on /v1/completions", + ) + if isinstance(logprobs, int) and not isinstance(logprobs, bool): + raise RequestError( + 400, + "invalid_logprobs", + "token logprobs are not supported on /v1/completions; pass false or omit", + ) + raise RequestError( + 400, + "invalid_logprobs", + "logprobs must be false; token logprobs are not supported on /v1/completions", + ) + + +def _validate_completions_top_logprobs(body: dict[str, Any]) -> None: + """Reject non-zero ``top_logprobs`` on legacy Completions. + + OpenAI Completions historically used integer ``logprobs`` (0–5); modern + chat uses boolean ``logprobs`` + ``top_logprobs``. This gateway never returns + token logprobs on /v1/completions, so non-zero ``top_logprobs`` fails closed + with ``invalid_top_logprobs`` rather than opaque ``unknown_fields``. + Explicit JSON null or ``0`` is treat-as-omit (SDK optional default / no top alts). + """ + if "top_logprobs" not in body: + return + value = body.get("top_logprobs") + # Explicit JSON null or zero is treat-as-omit (SDK optional default). + if value is None or value == 0: + return + raise RequestError( + 400, + "invalid_top_logprobs", + "top_logprobs is not supported on /v1/completions", + ) + + +def _validate_completions_suffix(body: dict[str, Any]) -> str | None: + """Legacy Completions ``suffix`` — optional string; non-empty is not supported. + + OpenAI appends ``suffix`` after the model completion. This gateway does not + implement that insertion, so a non-empty suffix fails closed. Empty string + and omit remain valid. Non-string values and oversized strings still fail. + """ + if "suffix" not in body: + return None + suffix = body.get("suffix") + # Explicit JSON null is treat-as-omit (SDK optional default). + if suffix is None: + return None + if not isinstance(suffix, str): + raise RequestError(400, "invalid_suffix", "suffix must be a string") + if len(suffix) > 8_000: + raise RequestError(400, "invalid_suffix", "suffix must be at most 8000 characters") + if suffix: + raise RequestError( + 400, + "invalid_suffix", + "non-empty suffix is not supported on /v1/completions", + ) + return suffix + + +def _validate_completions_best_of(body: dict[str, Any]) -> int | None: + """Legacy Completions ``best_of`` — positive integer, ``best_of >= n``, max 1. + + OpenAI generates ``best_of`` candidates server-side and returns the top ``n``. + This gateway runs a single completion path, so ``best_of > 1`` fails closed + rather than silently returning one unranked candidate. ``best_of=1`` (and + omit) remain valid. Boolean ``True``/``False`` are rejected. + """ + if "best_of" not in body: + return None + best_of = body.get("best_of") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if best_of is None or (isinstance(best_of, str) and not best_of.strip()): + return None + if isinstance(best_of, bool) or not isinstance(best_of, int) or best_of < 1: + raise RequestError(400, "invalid_best_of", "best_of must be a positive integer") + if best_of > 128: + raise RequestError(400, "invalid_best_of", "best_of must be at most 128") + if best_of > 1: + raise RequestError( + 400, + "invalid_best_of", + "best_of greater than 1 is not supported on /v1/completions", + ) + n = body.get("n", 1) + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise RequestError(400, "invalid_n", "n must be a positive integer") + if best_of < n: + raise RequestError( + 400, + "invalid_best_of", + "best_of must be greater than or equal to n", + ) + return best_of + + +def _validate_completions_stream_options(body: dict[str, Any]) -> dict[str, Any] | None: + """Legacy Completions ``stream_options`` — object with boolean flags; requires stream=true. + + Mirrors OpenAI chat Completions: ``stream_options`` is only valid when streaming. + This gateway rejects Completions streaming, so a well-formed ``stream_options`` + still fails closed once ``stream`` is checked (or here if ``stream`` is not true). + Explicit JSON null flag values are treat-as-omit (SDK optional defaults). + """ + if "stream_options" not in body: + return None + opts = body.get("stream_options") + # Explicit JSON null or empty object is treat-as-omit (SDK optional default). + if opts is None: + return None + if not isinstance(opts, dict): + raise RequestError(400, "invalid_stream_options", "stream_options must be an object") + if not opts: + return None + # Drop null flag values (SDK optional defaults) before further checks. + opts = {key: value for key, value in opts.items() if value is not None} + if not opts: + return None + # All-false boolean flags are omit-equivalent no-ops (SDK optional defaults). + allowed_flags = {"include_usage", "include_obfuscation"} + if set(opts) <= allowed_flags and all(v is False for v in opts.values()): + return None + if body.get("stream") is not True: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options requires stream=true", + ) + allowed = {"include_usage", "include_obfuscation"} + unknown = sorted(set(opts) - allowed) + if unknown: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options contains unsupported fields", + {"fields": unknown}, + ) + if "include_usage" in opts and not isinstance(opts["include_usage"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_usage must be a boolean", + ) + if "include_obfuscation" in opts and not isinstance(opts["include_obfuscation"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_obfuscation must be a boolean", + ) + return opts + + + + +def _validate_chat_stream_options(body: dict[str, Any], stream: bool) -> dict[str, Any] | None: + """Chat Completions ``stream_options`` — requires stream=true; include_usage unsupported. + + Shape matches OpenAI (include_usage / include_obfuscation booleans). This + gateway's SSE route path does not emit a final usage chunk and does not + apply stream obfuscation, so include_usage/include_obfuscation=true fail closed. + Explicit JSON null flag values are treat-as-omit (SDK optional defaults). + """ + if "stream_options" not in body: + return None + opts = body.get("stream_options") + # Explicit JSON null or empty object is treat-as-omit (SDK optional default). + if opts is None: + return None + if not isinstance(opts, dict): + raise RequestError(400, "invalid_stream_options", "stream_options must be an object") + if not opts: + return None + # Drop null flag values (SDK optional defaults) before further checks. + opts = {key: value for key, value in opts.items() if value is not None} + if not opts: + return None + # All-false boolean flags are omit-equivalent no-ops (SDK optional defaults). + allowed_flags = {"include_usage", "include_obfuscation"} + if set(opts) <= allowed_flags and all(v is False for v in opts.values()): + return None + if stream is not True: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options requires stream=true on /v1/chat/completions", + ) + allowed = {"include_usage", "include_obfuscation"} + unknown = sorted(set(opts) - allowed) + if unknown: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options contains unsupported fields", + {"fields": unknown}, + ) + if "include_usage" in opts: + if not isinstance(opts["include_usage"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_usage must be a boolean", + ) + if opts["include_usage"] is True: + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_usage=true is not supported on /v1/chat/completions", + ) + if "include_obfuscation" in opts: + if not isinstance(opts["include_obfuscation"], bool): + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_obfuscation must be a boolean", + ) + if opts["include_obfuscation"] is True: + # SSE obfuscation is not applied by this gateway; fail closed. + raise RequestError( + 400, + "invalid_stream_options", + "stream_options.include_obfuscation=true is not supported on /v1/chat/completions", + ) + return opts + + def _reject_unknown_keys(body: dict[str, Any], allowed: set[str]) -> None: unknown = sorted(set(body) - allowed) if unknown: raise RequestError(400, "unknown_fields", "request contains unsupported fields", {"fields": unknown}) + +def _validate_responses_conversation_controls(body: dict[str, Any]) -> None: + """Fail closed on OpenAI conversation-control fields this gateway does not apply. + + ``previous_response_id``, ``conversation``, ``truncation``, ``include``, and + ``text`` are real OpenAI Responses controls. Accepting them as unknown fields + yields opaque 400s; named unsupported errors let buyers migrate cleanly. + Explicit JSON null or empty string for string fields is treat-as-omit + (SDK optional default). Empty include/text structures remain omit no-ops. + """ + def _present_nonempty(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str) and not value.strip(): + return False + return True + + if "previous_response_id" in body and _present_nonempty(body.get("previous_response_id")): + raise RequestError( + 400, + "invalid_previous_response_id", + "previous_response_id is not supported on /v1/responses", + ) + if "conversation" in body and _present_nonempty(body.get("conversation")): + raise RequestError( + 400, + "invalid_conversation", + "conversation is not supported on /v1/responses", + ) + if "truncation" in body and _present_nonempty(body.get("truncation")): + raise RequestError( + 400, + "invalid_truncation", + "truncation is not supported on /v1/responses", + ) + if "include" in body: + include = body.get("include") + # Explicit JSON null, empty array, or empty/whitespace string is treat-as-omit. + if ( + include is None + or (isinstance(include, list) and not include) + or (isinstance(include, str) and not include.strip()) + ): + pass + else: + raise RequestError( + 400, + "invalid_include", + "include is not supported on /v1/responses", + ) + if "text" in body: + text = body.get("text") + # Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit. + if ( + text is None + or (isinstance(text, dict) and not text) + or (isinstance(text, str) and not text.strip()) + ): + pass + else: + raise RequestError( + 400, + "invalid_text", + "text is not supported on /v1/responses", + ) + + +def _validate_responses_stream_options(body: dict[str, Any]) -> None: + """Responses ``stream_options`` — not supported (Responses streaming is off). + + OpenAI pairs stream_options with stream=true. This gateway rejects + stream=true on /v1/responses, so any present stream_options would be a + silent no-op; fail closed instead. Explicit JSON null (object or flag + values) is treat-as-omit. + """ + if "stream_options" not in body: + return + opts = body.get("stream_options") + # Explicit JSON null or empty object is treat-as-omit (SDK optional default). + if opts is None or (isinstance(opts, dict) and not opts): + return + if isinstance(opts, dict): + # Null flag values alone are omit-equivalent (SDK optional defaults). + non_null = {key: value for key, value in opts.items() if value is not None} + if not non_null: + return + # All-false allowed flags are also omit-equivalent. + allowed_flags = {"include_usage", "include_obfuscation"} + if set(non_null) <= allowed_flags and all(v is False for v in non_null.values()): + return + raise RequestError( + 400, + "invalid_stream_options", + "stream_options is not supported on /v1/responses (stream is not supported)", + ) + + def _validate_mode(mode: Any) -> str: if not isinstance(mode, str) or mode not in ALLOWED_MODES: raise RequestError(400, "invalid_mode", "mode must be auto, route, or conduct") return mode -def _validate_messages(messages: Any) -> list[dict[str, str]]: + +def _require_pool_model(orchestrator: Any, model_name: str) -> None: + """Fail closed when ``model_name`` is not served by any enabled agent. + + OpenAI clients treat ``model`` as the deployment they paid for. Silently + answering with a different pool agent hides capacity/routing mismatches. + """ + agents = getattr(orchestrator, "agents", None) or [] + for agent in agents: + if getattr(agent, "disabled", False): + continue + if getattr(agent, "model", None) == model_name: + return + raise RequestError( + 400, + "invalid_model", + f"model {model_name!r} is not available in the agent pool", + ) + + + +def _validate_message_content_parts(content: list[Any]) -> list[dict[str, Any]]: + """OpenAI multimodal content-parts array (text + image_url) for vision callers. + + Parts are shape-checked and returned for provider passthrough. Unsupported + part types fail closed with a named error so clients never believe audio or + other modalities were processed. Empty/whitespace text and image URLs fail + closed; bare-string ``image_url`` is normalized to ``{"url": ...}``; optional + ``detail`` must be auto/low/high when present. + """ + if not content: + raise RequestError( + 400, + "invalid_message_content", + "multipart content arrays must be non-empty", + ) + parts: list[dict[str, Any]] = [] + for part in content: + if not isinstance(part, dict): + raise RequestError( + 400, + "invalid_message_content", + "message content part must be an object", + ) + part_type = part.get("type") + if part_type == "text": + text = part.get("text") + if not isinstance(text, str): + raise RequestError( + 400, + "invalid_message_content", + "text content part requires a string text field", + ) + if not text.strip(): + raise RequestError( + 400, + "invalid_message_content", + "text content part text must be a non-empty string", + ) + parts.append(part) + elif part_type == "image_url": + image_url = part.get("image_url") + # OpenAI SDKs occasionally send image_url as a bare URL string. + if isinstance(image_url, str): + if not image_url.strip(): + raise RequestError( + 400, + "invalid_message_content", + "image_url content part requires a non-empty url string", + ) + image_url = {"url": image_url} + part = {**part, "image_url": image_url} + if not isinstance(image_url, dict): + raise RequestError( + 400, + "invalid_message_content", + "image_url content part requires image_url.url as a string", + ) + url = image_url.get("url") + if not isinstance(url, str) or not url.strip(): + raise RequestError( + 400, + "invalid_message_content", + "image_url content part requires image_url.url as a non-empty string", + ) + if "detail" in image_url: + detail = image_url.get("detail") + # Explicit null / empty string: treat as omit (SDK optional default). + if detail is None or (isinstance(detail, str) and not detail.strip()): + cleaned = {key: value for key, value in image_url.items() if key != "detail"} + part = {**part, "image_url": cleaned} + else: + if not isinstance(detail, str): + raise RequestError( + 400, + "invalid_message_content", + "image_url.detail must be a string", + ) + detail_normalized = detail.strip().lower() + if detail_normalized not in {"auto", "low", "high"}: + raise RequestError( + 400, + "invalid_message_content", + "image_url.detail must be one of auto, low, high", + ) + if detail != detail_normalized: + part = { + **part, + "image_url": {**image_url, "detail": detail_normalized}, + } + parts.append(part) + else: + raise RequestError( + 400, + "invalid_message_content", + "content part type must be text or image_url", + ) + return parts + + +def _reject_unknown_message_keys(message: dict[str, Any]) -> None: + """Fail closed on chat message keys outside the OpenAI surface we honor. + + Named ``unknown_message_fields`` (with the key list) beats silent strip on + the orchestration path or silent smuggle on tools passthrough. + """ + unknown = sorted(set(message) - ALLOWED_MESSAGE_KEYS) + if unknown: + raise RequestError( + 400, + "unknown_message_fields", + "message contains unsupported fields", + {"fields": unknown}, + ) + + +def _validate_chat_message_known_fields(body: dict[str, Any]) -> None: + """Reject unknown message keys and legacy function role before passthrough.""" + messages = body.get("messages") + if not isinstance(messages, list): + return + for message in messages: + if not isinstance(message, dict): + continue + role = message.get("role") + if isinstance(role, str) and role == "function": + raise RequestError( + 400, + "invalid_message_role", + "function role is not supported on /v1/chat/completions; use tool instead", + ) + _reject_unknown_message_keys(message) + + +def _validate_messages(messages: Any) -> list[dict[str, Any]]: if not isinstance(messages, list) or not messages: raise RequestError(400, "invalid_message", "messages must be a non-empty array") - validated: list[dict[str, str]] = [] + validated: list[dict[str, Any]] = [] for message in messages: if not isinstance(message, dict): raise RequestError(400, "invalid_message", "each message must be an object") role = message.get("role") content = message.get("content") - if not isinstance(role, str) or role not in ALLOWED_MESSAGE_ROLES or not isinstance(content, str): + if isinstance(role, str) and role == "developer": + # Newer OpenAI clients send developer in place of system; this gateway + # does not apply a separate developer plane — fail closed with migration. + raise RequestError( + 400, + "invalid_message_role", + "developer role is not supported on /v1/chat/completions; use system instead", + ) + if isinstance(role, str) and role == "function": + # Legacy Completions function-calling role; tool replaces it. + raise RequestError( + 400, + "invalid_message_role", + "function role is not supported on /v1/chat/completions; use tool instead", + ) + # Named error for unsupported keys — never silent strip or passthrough smuggle. + _reject_unknown_message_keys(message) + if not isinstance(role, str) or role not in ALLOWED_MESSAGE_ROLES: raise RequestError(400, "invalid_message", "message role or content is invalid") - validated.append({"role": role, "content": content}) + # OpenAI assistant tool turns often send content:null with tool_calls; treat + # explicit JSON null as empty string on assistant/tool (SDK optional default). + if content is None and role in {"assistant", "tool"}: + content = "" + if isinstance(content, list): + # Vision/omni callers send OpenAI content-parts arrays. Shape-check and + # passthrough text+image_url; other part types fail closed. + content = _validate_message_content_parts(content) + elif not isinstance(content, str): + raise RequestError(400, "invalid_message", "message role or content is invalid") + # User/system turns drive the prompt — empty string content is never applied. + # Multimodal arrays are non-empty after parts validation. + if role in {"user", "system"} and isinstance(content, str) and not content.strip(): + raise RequestError( + 400, + "invalid_message_content", + "user and system message content must be a non-empty string", + ) + entry: dict[str, Any] = {"role": role, "content": content} + if role == "tool": + # OpenAI tool messages bind results to a prior tool_call via tool_call_id. + tool_call_id = message.get("tool_call_id") + if not isinstance(tool_call_id, str) or not tool_call_id.strip(): + raise RequestError( + 400, + "invalid_message", + "tool messages require a non-empty tool_call_id string", + ) + if len(tool_call_id) > 128: + raise RequestError( + 400, + "invalid_message", + "tool_call_id must be at most 128 characters", + ) + entry["tool_call_id"] = tool_call_id + if "name" in message: + # OpenAI optional participant name on system/user/assistant (not tool). + msg_name = message.get("name") + # Explicit JSON null or empty/whitespace string is treat-as-omit + # (SDK optional default / blank participant). + if msg_name is None or (isinstance(msg_name, str) and not msg_name.strip()): + pass + else: + if role == "tool": + raise RequestError( + 400, + "invalid_message_name", + "name is not valid on tool role messages", + ) + if not isinstance(msg_name, str): + raise RequestError( + 400, + "invalid_message_name", + "message name must be a non-empty string", + ) + if len(msg_name) > 64: + raise RequestError( + 400, + "invalid_message_name", + "message name must be at most 64 characters", + ) + # OpenAI participant names are alphanumeric plus underscore/hyphen. + if not all(ch.isalnum() or ch in "_-" for ch in msg_name): + raise RequestError( + 400, + "invalid_message_name", + "message name must match [a-zA-Z0-9_-]", + ) + entry["name"] = msg_name + if "refusal" in message: + # OpenAI assistant refusal plane — null/empty omit; non-empty fails closed + # (this gateway does not surface or apply refusal content). + refusal = message.get("refusal") + if refusal is None or (isinstance(refusal, str) and not refusal.strip()): + pass + elif role != "assistant": + raise RequestError( + 400, + "invalid_message_refusal", + "refusal is only valid on assistant messages", + ) + elif not isinstance(refusal, str): + raise RequestError( + 400, + "invalid_message_refusal", + "refusal must be a string", + ) + else: + raise RequestError( + 400, + "invalid_message_refusal", + "non-empty refusal is not supported on /v1/chat/completions", + ) + if "annotations" in message: + # OpenAI message annotations — null/empty omit; non-empty fails closed. + annotations = message.get("annotations") + if annotations is None or (isinstance(annotations, list) and not annotations): + pass + else: + raise RequestError( + 400, + "invalid_message_annotations", + "non-empty annotations are not supported on /v1/chat/completions", + ) + if "audio" in message: + # OpenAI assistant audio payload — null/empty omit; non-empty fails closed + # (this text gateway has no speech plane on chat message history). + audio = message.get("audio") + if audio is None or (isinstance(audio, dict) and not audio): + pass + else: + raise RequestError( + 400, + "invalid_message_audio", + "non-empty message audio is not supported on /v1/chat/completions", + ) + if "function_call" in message: + # Legacy assistant function_call on messages — null/empty omit; non-empty + # fails closed (use tool_calls; body-level function_call is also rejected). + function_call = message.get("function_call") + if function_call is None or (isinstance(function_call, dict) and not function_call): + pass + else: + raise RequestError( + 400, + "invalid_message_function_call", + "non-empty message function_call is not supported on /v1/chat/completions; " + "use tool_calls instead", + ) + if "weight" in message: + # OpenAI fine-tune style message weight (0 or 1). Explicit null is + # treat-as-omit. 0/1 are honest no-ops (no fine-tune plane here). + # Other values fail closed so clients never believe weighting applied. + weight = message.get("weight") + if weight is None: + pass + elif isinstance(weight, bool) or not isinstance(weight, (int, float)): + raise RequestError( + 400, + "invalid_message_weight", + "message weight must be 0 or 1", + ) + elif float(weight) not in (0.0, 1.0): + raise RequestError( + 400, + "invalid_message_weight", + "message weight must be 0 or 1", + ) + if "prefix" in message: + # OpenAI partial-assistant / predicted-outputs style prefix flag. + # null/false are honest no-ops; true fails closed (no prefix plane). + prefix = message.get("prefix") + if prefix is None or prefix is False: + pass + elif prefix is True: + raise RequestError( + 400, + "invalid_message_prefix", + "message prefix=true is not supported on /v1/chat/completions", + ) + else: + raise RequestError( + 400, + "invalid_message_prefix", + "message prefix must be a boolean", + ) + validated.append(entry) + return validated + + +def _validate_chat_message_audio_function_call(body: dict[str, Any]) -> None: + """Message-level ``audio`` / ``function_call`` — null/empty omit; else fail closed. + + Runs before tools passthrough so multi-turn histories with SDK-default + null slots stay honest even when the body is proxied verbatim. + """ + messages = body.get("messages") + if not isinstance(messages, list): + return + for message in messages: + if not isinstance(message, dict): + continue + if "audio" in message: + audio = message.get("audio") + if audio is None or (isinstance(audio, dict) and not audio): + pass + else: + raise RequestError( + 400, + "invalid_message_audio", + "non-empty message audio is not supported on /v1/chat/completions", + ) + if "function_call" in message: + function_call = message.get("function_call") + if function_call is None or (isinstance(function_call, dict) and not function_call): + pass + else: + raise RequestError( + 400, + "invalid_message_function_call", + "non-empty message function_call is not supported on /v1/chat/completions; " + "use tool_calls instead", + ) + + +def _validate_chat_tool_message_ids(body: dict[str, Any]) -> None: + """Fail closed on role=tool messages missing a usable tool_call_id. + + Runs before tools passthrough so multi-turn tool results are shape-checked + even when the body is proxied verbatim to a single provider agent. + """ + messages = body.get("messages") + if not isinstance(messages, list): + return + for message in messages: + if not isinstance(message, dict): + continue + if message.get("role") != "tool": + continue + tool_call_id = message.get("tool_call_id") + if not isinstance(tool_call_id, str) or not tool_call_id.strip(): + raise RequestError( + 400, + "invalid_message", + "tool messages require a non-empty tool_call_id string", + ) + if len(tool_call_id) > 128: + raise RequestError( + 400, + "invalid_message", + "tool_call_id must be at most 128 characters", + ) + + +def _validate_chat_assistant_tool_calls(body: dict[str, Any]) -> None: + """OpenAI assistant ``tool_calls`` array shape on chat messages. + + Each entry must be a function tool call with non-empty ``id``, + ``function.name``, and string ``function.arguments`` (JSON text). + Explicit JSON null or empty ``tool_calls`` arrays are treat-as-omit. + Validated before passthrough so multi-turn tool histories fail closed. + """ + messages = body.get("messages") + if not isinstance(messages, list): + return + for message in messages: + if not isinstance(message, dict): + continue + if "tool_calls" not in message: + continue + if message.get("role") != "assistant": + raise RequestError( + 400, + "invalid_message", + "tool_calls is only valid on assistant messages", + ) + tool_calls = message.get("tool_calls") + # Explicit JSON null or empty array is treat-as-omit (SDK optional default / + # no-op history slot). Non-empty arrays are shape-checked below. + if tool_calls is None or (isinstance(tool_calls, list) and not tool_calls): + continue + if not isinstance(tool_calls, list): + raise RequestError( + 400, + "invalid_message", + "tool_calls must be a non-empty array", + ) + if len(tool_calls) > 128: + raise RequestError( + 400, + "invalid_message", + "tool_calls must contain at most 128 entries", + ) + for call in tool_calls: + if not isinstance(call, dict): + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry must be an object", + ) + call_id = call.get("id") + if not isinstance(call_id, str) or not call_id.strip(): + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry requires a non-empty id string", + ) + if len(call_id) > 128: + raise RequestError( + 400, + "invalid_message", + "each tool_calls id must be at most 128 characters", + ) + if call.get("type") != "function": + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry type must be function", + ) + function = call.get("function") + if not isinstance(function, dict): + raise RequestError( + 400, + "invalid_message", + "each tool_calls entry requires a function object", + ) + name = function.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.name must be a non-empty string", + ) + if len(name) > 64: + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.name must be at most 64 characters", + ) + if not all(ch.isalnum() or ch in "_-" for ch in name): + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.name must match [a-zA-Z0-9_-]", + ) + arguments = function.get("arguments") + if not isinstance(arguments, str): + raise RequestError( + 400, + "invalid_message", + "each tool_calls function.arguments must be a string", + ) + + +def _validate_openai_metadata(body: dict[str, Any]) -> dict[str, str] | None: + """OpenAI ``metadata`` — object of string pairs, at most 16 entries. + + Keys ≤64 characters; values ≤512 characters. Non-objects and non-string + entries fail closed so clients cannot store untyped junk that cost or + observability consumers would silently drop. + """ + if "metadata" not in body: + return None + metadata = body.get("metadata") + # Explicit JSON null is treat-as-omit (SDK optional default). + if metadata is None: + return None + if not isinstance(metadata, dict): + raise RequestError(400, "invalid_metadata", "metadata must be an object") + if len(metadata) > 16: + raise RequestError(400, "invalid_metadata", "metadata must contain at most 16 entries") + validated: dict[str, str] = {} + for key, value in metadata.items(): + if not isinstance(key, str): + raise RequestError(400, "invalid_metadata", "metadata keys must be strings") + if len(key) > 64: + raise RequestError(400, "invalid_metadata", "metadata keys must be at most 64 characters") + if not isinstance(value, str): + raise RequestError(400, "invalid_metadata", "metadata values must be strings") + if len(value) > 512: + raise RequestError( + 400, + "invalid_metadata", + "metadata values must be at most 512 characters", + ) + validated[key] = value return validated @@ -211,6 +1797,12 @@ def _validate_attribution(attribution: Any) -> dict[str, Any] | None: def _validate_routing(routing: Any) -> dict[str, Any] | None: + """OpenAI-adjacent routing hints for sync vs batch channel selection. + + Fail closed on shape so callers cannot smuggle non-boolean latency flags or + free-form priority values that RoutingPolicy would silently misread via + loose coercion (``bool(x)`` / ``str(x)``). + """ if routing is None: return None if not isinstance(routing, dict): @@ -221,6 +1813,22 @@ def _validate_routing(routing: Any) -> dict[str, Any] | None: channel = routing.get("channel") if channel is not None and channel not in {"sync", "batch"}: raise RequestError(400, "invalid_routing", "routing.channel must be sync or batch") + if "latency_tolerant" in routing: + latency_tolerant = routing.get("latency_tolerant") + if not isinstance(latency_tolerant, bool): + raise RequestError( + 400, + "invalid_routing", + "routing.latency_tolerant must be a boolean", + ) + if "priority" in routing: + priority = routing.get("priority") + if not isinstance(priority, str) or priority not in {"interactive", "normal", "bulk"}: + raise RequestError( + 400, + "invalid_routing", + "routing.priority must be one of interactive, normal, bulk", + ) return routing @@ -248,22 +1856,1145 @@ def _validate_batch_requests(body: dict[str, Any], expose_trace: bool) -> list[B def _validate_embeddings_inputs(body: dict[str, Any]) -> list[str]: - """Validate the embeddings batch inputs (accepts ``inputs`` or ``input``).""" + """Validate embeddings ``input``/``inputs`` for sync and batch paths. + + Accepts a non-empty string or a non-empty array of non-empty strings. + Blank items fail closed: empty vectors pollute semantic search and cost + rollups without giving buyers a usable meaning unit. + """ raw = body.get("inputs") if raw is None: raw = body.get("input") if isinstance(raw, str): raw = [raw] if not isinstance(raw, list) or not raw: - raise RequestError(400, "invalid_request", "input/inputs must be a non-empty array of strings") + raise RequestError( + 400, + "invalid_input", + "input/inputs must be a non-empty string or non-empty array of strings", + ) inputs: list[str] = [] for item in raw: if not isinstance(item, str): - raise RequestError(400, "invalid_request", "each embedding input must be a string") + raise RequestError(400, "invalid_input", "each embedding input must be a string") + if not item.strip(): + raise RequestError( + 400, + "invalid_input", + "each embedding input must be a non-empty string", + ) inputs.append(item) return inputs + +def _validate_chat_store(body: dict[str, Any]) -> bool | None: + """Chat Completions ``store`` — strict boolean; ``true`` is not supported. + + OpenAI can persist completions when ``store=true``. This gateway does not + implement that persistence surface, so ``store=true`` fails closed. + ``store=false`` and omit remain valid (explicit no-store is honest). + """ + if "store" not in body: + return None + store = body.get("store") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if store is None or (isinstance(store, str) and not store.strip()): + return None + if not isinstance(store, bool): + raise RequestError(400, "invalid_store", "store must be a boolean") + if store is True: + raise RequestError( + 400, + "invalid_store", + "store=true is not supported on /v1/chat/completions", + ) + return store + + + +def _validate_completions_tools_surface(body: dict[str, Any]) -> None: + """Reject chat-era tool fields on legacy Completions with a migration path. + + OpenAI Completions has no tools surface. Clients migrating from chat often + still send tools/tool_choice. Named unsupported errors beat opaque + unknown_fields for commercial honesty. + + Honest no-ops (omit-equivalent SDK defaults): + - empty ``tools: []`` + - empty ``functions: []`` + - ``parallel_tool_calls=false`` / null + - ``tool_choice`` none/auto/empty-string/empty-object/null + - ``function_call`` none/auto/empty-string/null + + Non-empty tools/functions, non-default tool_choice/function_call, or + ``parallel_tool_calls=true`` fail closed with a chat migration path. + """ + tools = body.get("tools") if "tools" in body else None + # Empty array and explicit JSON null are omit-equivalent SDK defaults. + if tools is None or (isinstance(tools, list) and not tools): + tools_present = False + else: + tools_present = "tools" in body + + functions = body.get("functions") if "functions" in body else None + if functions is None or (isinstance(functions, list) and not functions): + functions_present = False + else: + functions_present = "functions" in body + + parallel = body.get("parallel_tool_calls") if "parallel_tool_calls" in body else None + if parallel is False or parallel is None: + # false or explicit null are omit-equivalent SDK defaults (no-ops). + parallel_present = False + elif "parallel_tool_calls" in body: + # true or non-boolean — surface as tools unsupported (or type error below). + if not isinstance(parallel, bool): + raise RequestError( + 400, + "invalid_parallel_tool_calls", + "parallel_tool_calls must be a boolean", + ) + parallel_present = True + else: + parallel_present = False + + def _tool_control_present(key: str) -> bool: + if key not in body: + return False + value = body.get(key) + # null, empty string, empty object, none/auto (whitespace-padded) are omit-equivalent. + if value is None: + return False + if isinstance(value, str): + stripped = value.strip() + if not stripped or stripped in ("none", "auto"): + return False + if isinstance(value, dict) and not value: + return False + return True + + if ( + tools_present + or functions_present + or parallel_present + or _tool_control_present("tool_choice") + or _tool_control_present("function_call") + ): + raise RequestError( + 400, + "invalid_tools", + "tools, tool_choice, functions, function_call, and parallel_tool_calls " + "are not supported on /v1/completions; use /v1/chat/completions instead", + ) + + +def _validate_completions_response_format_surface(body: dict[str, Any]) -> None: + """Reject response_format on legacy Completions with a migration path. + + Structured outputs are a chat/Responses surface. Completions has no + response_format plane — fail closed so clients migrate to chat. + """ + if "response_format" in body: + fmt = body.get("response_format") + # Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit. + if ( + fmt is None + or (isinstance(fmt, dict) and not fmt) + or (isinstance(fmt, str) and not fmt.strip()) + ): + return + raise RequestError( + 400, + "invalid_response_format", + "response_format is not supported on /v1/completions; use /v1/chat/completions instead", + ) + + +def _validate_completions_chat_era_fields_surface(body: dict[str, Any]) -> None: + """Reject chat-era modalities/prediction/reasoning_effort on Completions. + + Legacy Completions has no multi-modal output, Predicted Outputs, or o-series + reasoning_effort plane. Named unsupported errors beat opaque unknown_fields + so clients migrate to /v1/chat/completions. + Explicit JSON null, empty list/object, or empty/whitespace string is treat-as-omit. + """ + for key in ("modalities", "prediction", "reasoning_effort"): + if key not in body: + continue + value = body.get(key) + # Explicit JSON null, empty list/object, or empty string is treat-as-omit. + if value is None: + continue + if isinstance(value, (list, dict)) and not value: + continue + if isinstance(value, str) and not value.strip(): + continue + # Text-only modalities ["text"] is an honest no-op on this text gateway + # (parity with chat Completions allowing modalities ["text"]). + if key == "modalities" and isinstance(value, list): + stripped_items = [ + item.strip() if isinstance(item, str) else item for item in value + ] + if stripped_items == ["text"]: + continue + # reasoning_effort "none" disables extra reasoning — omit-equivalent no-op. + if key == "reasoning_effort" and isinstance(value, str) and value.strip() == "none": + continue + raise RequestError( + 400, + "invalid_chat_era_field", + "modalities, prediction, and reasoning_effort are not supported on " + "/v1/completions; use /v1/chat/completions instead", + ) + + +def _validate_completions_store(body: dict[str, Any]) -> bool | None: + """Legacy Completions ``store`` — strict boolean; ``true`` is not supported. + + OpenAI may persist completions when ``store=true``. This gateway has no + Completions persistence surface, so ``store=true`` fails closed rather than + silently ignoring a buyer-visible storage control. ``store=false``/omit stay valid. + """ + if "store" not in body: + return None + store = body.get("store") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if store is None or (isinstance(store, str) and not store.strip()): + return None + if not isinstance(store, bool): + raise RequestError(400, "invalid_store", "store must be a boolean") + if store is True: + raise RequestError( + 400, + "invalid_store", + "store=true is not supported on /v1/completions", + ) + return store + + +def _validate_responses_store(body: dict[str, Any]) -> bool | None: + """Responses API ``store`` — strict boolean; ``true`` is not supported. + + OpenAI may persist Responses when ``store=true``. This gateway's Responses + path is a single-agent passthrough without a persistence plane, so + ``store=true`` fails closed rather than silently dropping a buyer-visible + storage control. ``store=false`` and omit remain valid. + """ + if "store" not in body: + return None + store = body.get("store") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if store is None or (isinstance(store, str) and not store.strip()): + return None + if not isinstance(store, bool): + raise RequestError(400, "invalid_store", "store must be a boolean") + if store is True: + raise RequestError( + 400, + "invalid_store", + "store=true is not supported on /v1/responses", + ) + return store + + + + +def _validate_chat_reasoning_effort(body: dict[str, Any]) -> None: + """Chat Completions ``reasoning_effort`` — not applied on multi-agent route. + + OpenAI o-series models accept ``reasoning_effort`` (e.g. none/low/medium/high). + This gateway never threads the knob into ``ModelClient`` on the orchestration + path, so non-default present values fail closed rather than silently ignoring + a buyer-visible reasoning control. + + Explicit JSON null, empty/whitespace string, or ``none`` (whitespace-padded) + is treat-as-omit — ``none`` disables extra reasoning and is an honest no-op + here. + """ + if "reasoning_effort" not in body: + return + effort = body.get("reasoning_effort") + if effort is None: + return + if isinstance(effort, str): + stripped = effort.strip() + if not stripped or stripped == "none": + return + raise RequestError( + 400, + "invalid_reasoning_effort", + "reasoning_effort is not supported on /v1/chat/completions", + ) + + + + + +def _validate_chat_audio_web_search_surface( + body: dict[str, Any], + *, + endpoint_path: str = "/v1/chat/completions", +) -> None: + """Reject ``audio`` / ``web_search_options`` with named migration errors. + + This text gateway has no speech synthesis plane and no web-search tool + harness on chat or Completions. Named unsupported errors beat opaque + ``unknown_fields`` so SDK clients can migrate deliberately. + Explicit JSON null or empty object for either field is treat-as-omit + (SDK optional default). + """ + if "audio" in body: + audio = body.get("audio") + if audio is not None and not (isinstance(audio, dict) and not audio): + raise RequestError( + 400, + "invalid_audio", + f"audio is not supported on {endpoint_path}", + ) + if "web_search_options" in body: + web = body.get("web_search_options") + if web is not None and not (isinstance(web, dict) and not web): + raise RequestError( + 400, + "invalid_web_search_options", + f"web_search_options is not supported on {endpoint_path}", + ) + + + +def _validate_openai_sdk_control_fields(body: dict[str, Any], *, endpoint_path: str) -> None: + """Reject modern OpenAI SDK control fields not applied on this gateway. + + ``prompt_cache_key``, ``safety_identifier``, ``verbosity``, and ``prompt_cache_retention`` appear in + recent OpenAI SDK clients. This gateway has no prompt-cache affinity plane, + no safety-identifier side channel, and no verbosity sampling control — named + unsupported errors beat opaque ``unknown_fields``. + """ + def _sdk_control_present(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str) and not value.strip(): + return False + return True + + # Explicit JSON null or empty string is treat-as-omit (SDK optional default). + if "prompt_cache_key" in body and _sdk_control_present(body.get("prompt_cache_key")): + raise RequestError( + 400, + "invalid_prompt_cache_key", + f"prompt_cache_key is not supported on {endpoint_path}", + ) + if "safety_identifier" in body and _sdk_control_present(body.get("safety_identifier")): + raise RequestError( + 400, + "invalid_safety_identifier", + f"safety_identifier is not supported on {endpoint_path}", + ) + if "verbosity" in body and _sdk_control_present(body.get("verbosity")): + raise RequestError( + 400, + "invalid_verbosity", + f"verbosity is not supported on {endpoint_path}", + ) + if "prompt_cache_retention" in body and _sdk_control_present(body.get("prompt_cache_retention")): + raise RequestError( + 400, + "invalid_prompt_cache_retention", + f"prompt_cache_retention is not supported on {endpoint_path}", + ) + + + +def _validate_chat_reasoning_object(body: dict[str, Any]) -> None: + """Reject Responses-style ``reasoning`` object on chat Completions. + + OpenAI Responses accepts a ``reasoning`` object; chat Completions uses + ``reasoning_effort`` (already fail-closed). Clients that send ``reasoning`` + on chat must get a named error, not opaque unknown_fields. + Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit + (SDK optional default / stringified empty control). + """ + if "reasoning" not in body: + return + value = body.get("reasoning") + # Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit. + if ( + value is None + or (isinstance(value, dict) and not value) + or (isinstance(value, str) and not value.strip()) + ): + return + raise RequestError( + 400, + "invalid_reasoning", + "reasoning is not supported on /v1/chat/completions; use /v1/responses or omit", + ) + + + +def _validate_openai_background(body: dict[str, Any], *, endpoint_path: str) -> bool | None: + """OpenAI ``background`` — ``false``/omit are honest no-ops; ``true`` fails closed. + + OpenAI may run long jobs asynchronously when ``background=true``. This + gateway is request-scoped with no background job plane, so ``true`` fails + closed. ``false`` is a deliberate no-op (SDK defaults often send it). + """ + if "background" not in body: + return None + value = body.get("background") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if value is None or (isinstance(value, str) and not value.strip()): + return None + if not isinstance(value, bool): + raise RequestError(400, "invalid_background", "background must be a boolean") + if value is True: + raise RequestError( + 400, + "invalid_background", + f"background=true is not supported on {endpoint_path}", + ) + return False + + + +def _validate_chat_include_field(body: dict[str, Any], *, endpoint_path: str = "/v1/chat/completions") -> None: + """Reject OpenAI ``include`` outside Responses (where it is also unsupported). + + Some SDKs send ``include`` on chat/Completions. Named error beats opaque + unknown_fields so clients know the surface is unsupported here. + Explicit JSON null, empty array, or empty/whitespace string is treat-as-omit. + """ + if "include" not in body: + return + include = body.get("include") + # Explicit JSON null, empty array, or empty/whitespace string is treat-as-omit. + if ( + include is None + or (isinstance(include, list) and not include) + or (isinstance(include, str) and not include.strip()) + ): + return + raise RequestError( + 400, + "invalid_include", + f"include is not supported on {endpoint_path}", + ) + + +def _validate_completions_reasoning_object(body: dict[str, Any]) -> None: + """Reject Responses-style ``reasoning`` object on legacy Completions. + + Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit + (SDK optional default / stringified empty control). + """ + if "reasoning" not in body: + return + value = body.get("reasoning") + if ( + value is None + or (isinstance(value, dict) and not value) + or (isinstance(value, str) and not value.strip()) + ): + return + raise RequestError( + 400, + "invalid_reasoning", + "reasoning is not supported on /v1/completions; use /v1/responses or omit", + ) + + +def _validate_responses_modalities(body: dict[str, Any]) -> list[str] | None: + """Responses ``modalities`` — omit or ``["text"]`` only (text gateway).""" + if "modalities" not in body: + return None + modalities = body.get("modalities") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if modalities is None or (isinstance(modalities, str) and not modalities.strip()): + return None + if not isinstance(modalities, list): + raise RequestError( + 400, + "invalid_modalities", + "modalities must be a non-empty array of strings", + ) + # Empty array is omit-equivalent (SDK optional default). + if not modalities: + return None + if any(not isinstance(item, str) for item in modalities): + raise RequestError( + 400, + "invalid_modalities", + "modalities must be a non-empty array of strings", + ) + # Strip incidental whitespace on items so [" text "] matches text-only. + modalities = [item.strip() for item in modalities] + if modalities != ["text"]: + raise RequestError( + 400, + "invalid_modalities", + 'only modalities ["text"] is supported on /v1/responses', + ) + return modalities + + +def _validate_responses_prediction(body: dict[str, Any]) -> None: + """Responses ``prediction`` (Predicted Outputs) — not supported on this gateway. + + Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit. + """ + if "prediction" not in body: + return + value = body.get("prediction") + if ( + value is None + or (isinstance(value, dict) and not value) + or (isinstance(value, str) and not value.strip()) + ): + return + raise RequestError( + 400, + "invalid_prediction", + "prediction is not supported on /v1/responses", + ) + + +def _validate_chat_modalities(body: dict[str, Any]) -> list[str] | None: + """Chat Completions ``modalities`` — omit or ``["text"]`` only. + + OpenAI selects output types (text/audio) via modalities. This gateway is + text-only; non-text modalities fail closed so clients cannot silently + believe audio (or other) output was applied. + """ + if "modalities" not in body: + return None + modalities = body.get("modalities") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if modalities is None or (isinstance(modalities, str) and not modalities.strip()): + return None + if not isinstance(modalities, list): + raise RequestError( + 400, + "invalid_modalities", + "modalities must be a non-empty array of strings", + ) + # Empty array is omit-equivalent (SDK optional default). + if not modalities: + return None + if any(not isinstance(item, str) for item in modalities): + raise RequestError( + 400, + "invalid_modalities", + "modalities must be a non-empty array of strings", + ) + # Strip incidental whitespace on items so [" text "] matches text-only. + modalities = [item.strip() for item in modalities] + if modalities != ["text"]: + raise RequestError( + 400, + "invalid_modalities", + 'only modalities ["text"] is supported on /v1/chat/completions', + ) + return modalities + + +def _validate_chat_prediction(body: dict[str, Any]) -> None: + """Chat Completions ``prediction`` (Predicted Outputs) — not supported. + + OpenAI Predicted Outputs lets clients supply expected completion content for + latency wins. This gateway does not apply ``prediction`` on the multi-agent + route path, so any non-empty present value fails closed rather than silently + ignoring a buyer-visible optimization hint. + Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit. + """ + if "prediction" not in body: + return + value = body.get("prediction") + if ( + value is None + or (isinstance(value, dict) and not value) + or (isinstance(value, str) and not value.strip()) + ): + return + raise RequestError( + 400, + "invalid_prediction", + "prediction is not supported on /v1/chat/completions", + ) + + +def _validate_chat_response_format(body: dict[str, Any]) -> dict[str, Any] | None: + """OpenAI chat ``response_format`` — object with type text/json_object/json_schema. + + Shape is validated before passthrough so malformed payloads fail closed + rather than reaching a provider with an unusable format object. + + OpenAI type-only forms are strict: ``text`` and ``json_object`` accept only + the ``type`` key. ``json_schema`` accepts only ``type`` and ``json_schema``. + Extra sibling keys fail closed so clients cannot smuggle unsupported fields + into a provider-shaped object that this gateway never interpreted. + Inside ``json_schema``, only ``name``, ``description``, ``schema``, and + ``strict`` are accepted. ``name`` must match ``[a-zA-Z0-9_-]{1,64}`` + (ASCII only — ``str.isalnum()`` is not sufficient). + JSON-null or blank ``description`` / null ``strict`` are popped in + place so passthrough matches omit. + """ + if "response_format" not in body: + return None + fmt = body.get("response_format") + # Explicit JSON null, empty object, or empty string is treat-as-omit + # (SDK optional default / stringified empty control). + if ( + fmt is None + or (isinstance(fmt, dict) and not fmt) + or (isinstance(fmt, str) and not fmt.strip()) + ): + return None + if not isinstance(fmt, dict): + raise RequestError( + 400, + "invalid_response_format", + "response_format must be an object", + ) + fmt_type = fmt.get("type") + if fmt_type not in ("text", "json_object", "json_schema"): + raise RequestError( + 400, + "invalid_response_format", + "response_format.type must be one of text, json_object, json_schema", + ) + if fmt_type in ("text", "json_object"): + # OpenAI: {"type": "json_object"} / {"type": "text"} — no siblings. + unknown = sorted(set(fmt) - {"type"}) + if unknown: + raise RequestError( + 400, + "invalid_response_format", + f"response_format with type {fmt_type} accepts only the type field", + {"fields": unknown}, + ) + return fmt + if fmt_type == "json_schema": + unknown = sorted(set(fmt) - {"type", "json_schema"}) + if unknown: + raise RequestError( + 400, + "invalid_response_format", + "response_format with type json_schema accepts only type and json_schema", + {"fields": unknown}, + ) + schema = fmt.get("json_schema") + if not isinstance(schema, dict): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema must be an object when type is json_schema", + ) + # OpenAI json_schema objects are name + schema + optional description/strict. + # Extra siblings fail closed so clients cannot smuggle uninterpreted + # fields through passthrough (including null vendor extensions). + unknown_schema = sorted( + set(schema) - {"name", "description", "schema", "strict"} + ) + if unknown_schema: + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema accepts only name, description, schema, and strict", + {"fields": unknown_schema}, + ) + name = schema.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.name must be a non-empty string", + ) + # OpenAI Structured Outputs: name is [a-zA-Z0-9_-]{1,64}. Fail closed + # so buyers get invalid_response_format instead of a provider 400. + # str.isalnum() alone accepts Unicode letters/digits (café, 名前, ١٢٣). + if len(name) > 64: + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.name must be at most 64 characters", + ) + if not name.isascii() or not all(ch.isalnum() or ch in "_-" for ch in name): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.name must match [a-zA-Z0-9_-]", + ) + # OpenAI requires json_schema.schema as the actual JSON Schema object. + # Fail closed when missing or non-object so clients cannot silently + # believe structured-output enforcement applied without a schema body. + schema_body = schema.get("schema") + if not isinstance(schema_body, dict): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.schema must be an object", + ) + # SDK optional defaults serialize omitted description as JSON null or + # blank. Pop in place so proxy_completion forwards omit, not null. + if "description" in schema: + description_value = schema.get("description") + if description_value is None or ( + isinstance(description_value, str) and not description_value.strip() + ): + schema.pop("description") + elif not isinstance(description_value, str): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.description must be a string when provided", + ) + # Explicit JSON null is omit-equivalent: pop so passthrough matches omit. + if "strict" in schema: + strict_value = schema.get("strict") + if strict_value is None: + schema.pop("strict") + elif not isinstance(strict_value, bool): + raise RequestError( + 400, + "invalid_response_format", + "response_format.json_schema.strict must be a boolean when provided", + ) + return fmt + + +def _omit_null_tool_function_field( + function: dict[str, Any], + field_name: str, + *, + expected_types: tuple[type, ...], + error_message: str, +) -> None: + """Drop a JSON-null optional ``tool.function`` field or fail-closed. + + Official OpenAI SDKs serialize omitted optional fields as JSON ``null``. + Leaving those keys on the body is not omit-equivalent: ``proxy_completion`` + forwards the request verbatim and several providers reject ``null`` + ``parameters``, ``description``, or ``strict``. Pop the key in place so the + upstream payload matches an omitted field. Blank ``description`` strings + are popped by the caller after this helper. Non-null values of the wrong + type stay ``invalid_tools``. + """ + if field_name not in function: + return + value = function.get(field_name) + if value is None: + function.pop(field_name) + return + if not isinstance(value, expected_types): + raise RequestError(400, "invalid_tools", error_message) + + +def _validate_chat_tools(body: dict[str, Any]) -> list[dict[str, Any]] | None: + """OpenAI chat ``tools`` — array of function tool objects (empty = honest no-op). + + An empty array is treated as omit: many SDKs send ``tools: []`` when no tools + are configured. Non-empty entries must be objects with ``type`` == ``function`` + and a ``function`` object that has a non-empty ``name`` matching + ``[a-zA-Z0-9_-]{1,64}`` (ASCII only — ``str.isalnum()`` is not + sufficient). Explicit JSON ``null`` on optional ``description``, + ``parameters``, and ``strict`` is popped in place so passthrough + matches omit. Empty or whitespace-only ``description`` is also + popped. Shape-only validation before passthrough; provider schema + depth is not re-checked here. + """ + if "tools" not in body: + return None + tools = body.get("tools") + # Explicit JSON null is treat-as-omit (SDK optional default). + if tools is None: + return None + if not isinstance(tools, list): + raise RequestError( + 400, + "invalid_tools", + "tools must be an array", + ) + # Empty array: honest no-op (same as omitting tools). + if not tools: + return [] + if len(tools) > 128: + raise RequestError( + 400, + "invalid_tools", + "tools must contain at most 128 entries", + ) + validated: list[dict[str, Any]] = [] + for item in tools: + if not isinstance(item, dict): + raise RequestError(400, "invalid_tools", "each tool must be an object") + # OpenAI tool objects are type + function only; extra siblings fail closed + # so clients cannot smuggle uninterpreted fields through passthrough. + unknown_tool = sorted(set(item) - {"type", "function"}) + if unknown_tool: + raise RequestError( + 400, + "invalid_tools", + "each tool accepts only type and function fields", + {"fields": unknown_tool}, + ) + if item.get("type") != "function": + raise RequestError( + 400, + "invalid_tools", + "each tool type must be function", + ) + function = item.get("function") + if not isinstance(function, dict): + raise RequestError( + 400, + "invalid_tools", + "each tool.function must be an object", + ) + unknown_fn = sorted(set(function) - {"name", "description", "parameters", "strict"}) + if unknown_fn: + raise RequestError( + 400, + "invalid_tools", + "each tool.function accepts only name, description, parameters, and strict", + {"fields": unknown_fn}, + ) + # Explicit JSON null is popped so proxy_completion forwards omit, not null. + _omit_null_tool_function_field( + function, + "strict", + expected_types=(bool,), + error_message="each tool.function.strict must be a boolean when provided", + ) + name = function.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_tools", + "each tool.function.name must be a non-empty string", + ) + # OpenAI function names: [a-zA-Z0-9_-]{1,64}. Fail closed so buyers + # get invalid_tools instead of a provider 400. + # str.isalnum() alone accepts Unicode letters/digits (café, 名前, ١٢٣). + if len(name) > 64: + raise RequestError( + 400, + "invalid_tools", + "each tool.function.name must be at most 64 characters", + ) + if not name.isascii() or not all(ch.isalnum() or ch in "_-" for ch in name): + raise RequestError( + 400, + "invalid_tools", + "each tool.function.name must match [a-zA-Z0-9_-]", + ) + # OpenAI function tools require parameters as a JSON Schema object when present. + # Explicit JSON null is popped so proxy_completion forwards omit, not null. + _omit_null_tool_function_field( + function, + "parameters", + expected_types=(dict,), + error_message="each tool.function.parameters must be an object", + ) + _omit_null_tool_function_field( + function, + "description", + expected_types=(str,), + error_message="each tool.function.description must be a string when provided", + ) + description = function.get("description") + # Empty/whitespace is omit-equivalent (SDK optional default). + if isinstance(description, str) and not description.strip(): + function.pop("description") + description = None + if isinstance(description, str) and len(description) > 1024: + raise RequestError( + 400, + "invalid_tools", + "each tool.function.description must be at most 1024 characters", + ) + validated.append(item) + return validated + + +def _validate_chat_tool_choice(body: dict[str, Any]) -> str | dict[str, Any] | None: + """OpenAI chat ``tool_choice`` — none/auto/required or named function object. + + When ``type`` is ``function``, ``function.name`` must match a tools entry + so clients cannot force a tool the request did not declare. + """ + if "tool_choice" not in body: + return None + choice = body.get("tool_choice") + # Explicit JSON null, empty object, or empty/whitespace string is + # treat-as-omit (SDK optional default / stringified empty control). + if ( + choice is None + or (isinstance(choice, dict) and not choice) + or (isinstance(choice, str) and not choice.strip()) + ): + return None + if isinstance(choice, str): + # Strip incidental whitespace so " none " / " auto " match honest no-ops. + choice = choice.strip() + if choice not in ("none", "auto", "required"): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice string must be one of none, auto, required", + ) + return choice + if isinstance(choice, dict): + # OpenAI named tool_choice is {type, function}; extra siblings fail closed. + unknown = sorted(set(choice) - {"type", "function"}) + if unknown: + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice object accepts only type and function fields", + {"fields": unknown}, + ) + if choice.get("type") != "function": + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice object type must be function", + ) + function = choice.get("function") + if not isinstance(function, dict): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function must be an object with a name", + ) + unknown_fn = sorted(set(function) - {"name"}) + if unknown_fn: + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function accepts only name", + {"fields": unknown_fn}, + ) + name = function.get("name") + if not isinstance(name, str) or not name.strip(): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function.name must be a non-empty string", + ) + tools = body.get("tools") + tool_names: set[str] = set() + if isinstance(tools, list): + for item in tools: + if not isinstance(item, dict): + continue + fn = item.get("function") + if isinstance(fn, dict): + tool_name = fn.get("name") + if isinstance(tool_name, str): + tool_names.add(tool_name) + if name not in tool_names: + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice.function.name must match a tools entry", + ) + return choice + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice must be a string or object", + ) + + + + + + +def _validate_responses_model(body: dict[str, Any]) -> str: + """Responses API ``model`` — required non-empty string ≤256 chars. + + OpenAI requires model on Responses. Missing/empty/non-string values fail + closed so clients cannot hit passthrough with an implicit mock default and + believe a named deployment was selected. + """ + model = body.get("model") + if model is None: + raise RequestError(400, "invalid_model", "model is required on /v1/responses") + if not isinstance(model, str) or not model.strip(): + raise RequestError(400, "invalid_model", "model must be a non-empty string") + model = model.strip() + if len(model) > 256: + raise RequestError(400, "invalid_model", "model must be at most 256 characters") + return model + + +def _validate_responses_instructions(body: dict[str, Any]) -> str | None: + """Responses API ``instructions`` — optional non-empty string ≤32000 chars. + + OpenAI system-style instructions for the Responses surface. Empty strings + and non-strings fail closed so clients cannot ship a silent no-op that + looks like a configured system prompt. + """ + if "instructions" not in body: + return None + value = body.get("instructions") + # Explicit JSON null is treat-as-omit (SDK optional default). + if value is None: + return None + if not isinstance(value, str): + raise RequestError(400, "invalid_instructions", "instructions must be a string") + if not value.strip(): + raise RequestError( + 400, + "invalid_instructions", + "instructions must be a non-empty string on /v1/responses", + ) + if len(value) > 32_000: + raise RequestError( + 400, + "invalid_instructions", + "instructions must be at most 32000 characters", + ) + return value + + +def _validate_responses_reasoning(body: dict[str, Any]) -> None: + """Responses API ``reasoning`` — not applied on single-agent passthrough. + + OpenAI Responses accepts a ``reasoning`` object (effort/summary controls). + This gateway proxies Responses but does not interpret or enforce reasoning + controls, so any non-empty present value fails closed rather than silently + ignoring a buyer-visible o-series control surface. + Explicit JSON null, empty object, or empty/whitespace string is treat-as-omit. + """ + if "reasoning" not in body: + return + value = body.get("reasoning") + if ( + value is None + or (isinstance(value, dict) and not value) + or (isinstance(value, str) and not value.strip()) + ): + return + raise RequestError( + 400, + "invalid_reasoning", + "reasoning is not supported on /v1/responses", + ) + + + +def _validate_batch_embeddings_endpoint(body: dict[str, Any]) -> str | None: + """Batch embeddings ``endpoint`` — optional non-empty string alias ≤256 chars. + + naruon and OpenAI-compatible clients may tag the upstream embeddings route + (e.g. ``/v1/embeddings``). Explicit JSON null or empty/whitespace string is + treat-as-omit (SDK optional default). Non-string values fail closed so the + gateway never records a blank endpoint alias as if a route was selected. + """ + if "endpoint" not in body: + return None + value = body.get("endpoint") + # Explicit JSON null or empty/whitespace string is treat-as-omit. + if value is None or (isinstance(value, str) and not value.strip()): + return None + if not isinstance(value, str): + raise RequestError( + 400, + "invalid_endpoint", + "endpoint must be a non-empty string on /v1/batch/embeddings", + ) + if len(value) > 256: + raise RequestError( + 400, + "invalid_endpoint", + "endpoint must be at most 256 characters", + ) + return value + + +def _validate_embeddings_model(body: dict[str, Any]) -> str: + """OpenAI embeddings ``model`` — required non-empty string ≤256 chars.""" + model = body.get("model") + if model is None: + raise RequestError(400, "invalid_model", "model is required") + if not isinstance(model, str) or not model.strip(): + raise RequestError(400, "invalid_model", "model must be a non-empty string") + model = model.strip() + if len(model) > 256: + raise RequestError(400, "invalid_model", "model must be at most 256 characters") + return model + + +def _validate_embeddings_encoding_format(body: dict[str, Any]) -> str | None: + """OpenAI ``encoding_format`` — omit/null/empty or ``float`` only; base64 fail-closed. + + This gateway returns float vectors on the OpenAI list shape. ``base64`` is + not produced, so requesting it fails closed rather than silently returning + floats. Explicit JSON ``null`` or empty/whitespace string is treated as omit + (SDK optional default / stringified empty control). + """ + if "encoding_format" not in body: + return None + value = body.get("encoding_format") + if value is None or (isinstance(value, str) and not value.strip()): + return None + if not isinstance(value, str): + raise RequestError(400, "invalid_encoding_format", "encoding_format must be a string") + # Strip incidental whitespace so " float " matches float. + value = value.strip() + if value != "float": + raise RequestError( + 400, + "invalid_encoding_format", + 'only encoding_format "float" is supported on embeddings endpoints', + ) + return value + + +def _validate_embeddings_dimensions(body: dict[str, Any]) -> None: + """OpenAI ``dimensions`` — not applied; non-null values fail closed. + + Explicit JSON ``null`` or empty/whitespace string is treat-as-omit. Any other + value fails closed so clients cannot believe reduced dimensionality was applied. + """ + if "dimensions" not in body: + return + value = body.get("dimensions") + if value is None or (isinstance(value, str) and not value.strip()): + return + raise RequestError( + 400, + "invalid_dimensions", + "dimensions is not supported on embeddings endpoints", + ) + + +def _openai_embeddings_response(document: dict[str, Any], *, model: str) -> dict[str, Any]: + """Map batch document vectors to the OpenAI ``/v1/embeddings`` list shape.""" + items = document.get("embeddings") or [] + data = [] + for item in items: + data.append( + { + "object": "embedding", + "index": int(item.get("index", 0)), + "embedding": list(item.get("embedding") or []), + } + ) + total_tokens = int(document.get("total_tokens") or 0) + return { + "object": "list", + "data": data, + "model": model or document.get("model") or "contextual-orchestrator", + "usage": { + "prompt_tokens": total_tokens, + "total_tokens": total_tokens, + }, + } + + def _embeddings_attribution(body: dict[str, Any]) -> dict[str, Any]: """Build ledger attribution from the explicit ``attribution`` field merged with any attribution dimensions carried inside ``metadata``. @@ -346,6 +3077,20 @@ def do_GET(self) -> None: # noqa: N802 "usage_record_count": len(coordinator.ledger.records()), }) return + if path == "/v1/models" or path.startswith("/v1/models/"): + # OpenAI model discovery is inference-scope (same bearer as chat). + self._authorize("inference") + if path == "/v1/models": + self._send(orchestrator.list_openai_models()) + return + model_id = urllib.parse.unquote(path[len("/v1/models/") :]) + if not model_id or "/" in model_id: + raise RequestError(400, "invalid_model", "model id path must be a single segment") + try: + self._send(orchestrator.get_openai_model(model_id)) + except KeyError: + self._send_error(404, "model_not_found", f"model {model_id!r} not found") + return if path.startswith("/v1/batch/embeddings/"): # Embeddings batch polling is an inference-scope surface, so # it is authorized here before the admin gate below. @@ -711,9 +3456,228 @@ def do_POST(self) -> None: # noqa: N802 self._send(orchestrator.add_agent(segments[3], body), 201) return + if path == "/v1/completions": + # Legacy OpenAI Completions: prompt → route → text_completion. + _reject_unknown_keys(body, ALLOWED_COMPLETIONS_KEYS) + _validate_completions_tools_surface(body) + _validate_completions_response_format_surface(body) + _validate_completions_chat_era_fields_surface(body) + _validate_chat_audio_web_search_surface( + body, endpoint_path="/v1/completions" + ) + _validate_openai_sdk_control_fields(body, endpoint_path="/v1/completions") + _validate_max_tool_calls(body, endpoint_path="/v1/completions") + _validate_completions_reasoning_object(body) + _validate_openai_background(body, endpoint_path="/v1/completions") + _validate_chat_include_field(body, endpoint_path="/v1/completions") + _validate_completions_stream(body) + _validate_completions_stream_options(body) + _validate_completions_best_of(body) + _validate_completions_echo(body) + _validate_completions_suffix(body) + _validate_completions_logprobs(body) + _validate_completions_top_logprobs(body) + # OpenAI chat-era clients sometimes send max_completion_tokens + # on Completions; prefer it over legacy max_tokens when both set. + if "max_completion_tokens" in body: + max_tokens = _validate_chat_max_completion_tokens(body) + else: + max_tokens = _validate_completions_max_tokens(body) + model_name = _validate_completions_model(body) + _require_pool_model(orchestrator, model_name) + if "store" in body: + _validate_completions_store(body) + top_p = _validate_completions_top_p(body) + temperature = _validate_completions_temperature(body) + presence_penalty = _validate_completions_presence_penalty(body) + frequency_penalty = _validate_completions_frequency_penalty(body) + _validate_completions_seed(body) + _validate_completions_stop(body) + _validate_completions_n(body) + end_user_id = _validate_completions_user(body) + _validate_completions_logit_bias(body) + _validate_service_tier(body, endpoint_path="/v1/completions") + if "metadata" in body: + _validate_openai_metadata(body) + if "prompt" not in body: + raise RequestError(400, "invalid_prompt", "prompt is required") + messages = _validate_completion_prompt(body.get("prompt")) + attribution = _validate_attribution(body.get("attribution")) + attribution = dict(attribution or {}) + # OpenAI ``user`` → cost-ledger account when attribution.account is unset. + if end_user_id is not None and not attribution.get("account"): + attribution["account"] = end_user_id + # Request model id → model_name dimension when unset (cost rollups). + if model_name and not attribution.get("model_name"): + attribution["model_name"] = model_name + # Endpoint product surface → service dimension when unset. + if not attribution.get("service"): + attribution["service"] = "completions_api" + routing = _validate_routing(body.get("routing")) + started_at = time.perf_counter() + # Apply request sampling knobs to the provider client for this call. + model_client = orchestrator.client + previous_max_tokens = model_client.max_output_tokens + previous_temperature = model_client.default_temperature + previous_top_p = model_client.default_top_p + previous_presence = model_client.default_presence_penalty + previous_frequency = model_client.default_frequency_penalty + if max_tokens is not None: + model_client.max_output_tokens = max_tokens + if temperature is not None: + model_client.default_temperature = temperature + if top_p is not None: + model_client.default_top_p = top_p + if presence_penalty is not None: + model_client.default_presence_penalty = presence_penalty + if frequency_penalty is not None: + model_client.default_frequency_penalty = frequency_penalty + try: + result = self._run(lambda: coordinator.complete( + messages, + mode="route", + attribution=attribution, + hints=routing, + model_name=model_name, + workflow_run_id=f"run_{uuid.uuid4().hex}", + )) + finally: + model_client.max_output_tokens = previous_max_tokens + model_client.default_temperature = previous_temperature + model_client.default_top_p = previous_top_p + model_client.default_presence_penalty = previous_presence + model_client.default_frequency_penalty = previous_frequency + # Batch-channel Completions return a job handle (202), not a + # text_completion body — match chat Completions honesty so + # clients never receive a 500 on a valid batch routing hint. + if isinstance(result, dict) and result.get("channel") == "batch": + orchestrator.record_analytics_event( + "text_completion_batched", + { + "endpoint_path": "/v1/completions", + "actor_scope": "inference", + "status_code": 202, + "batch_job_id": result.get("job_id"), + "batch_backend": result.get("backend"), + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + }, + ) + self._send(result, 202) + return + orchestrator.record_analytics_event( + "text_completion_requested", + { + "endpoint_path": "/v1/completions", + "actor_scope": "inference", + "status_code": 200, + "run_mode": "route", + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + }, + ) + self._send(text_completion_response( + result, model=model_name, usage=result.get("usage"), + )) + return if path == "/v1/chat/completions": _reject_unknown_keys(body, ALLOWED_CHAT_KEYS) - if PASSTHROUGH_TRIGGER_KEYS & set(body): + _validate_chat_audio_web_search_surface(body) + _validate_openai_sdk_control_fields(body, endpoint_path="/v1/chat/completions") + _validate_chat_reasoning_object(body) + _validate_openai_background(body, endpoint_path="/v1/chat/completions") + _validate_chat_include_field(body) + _validate_max_tool_calls(body, endpoint_path="/v1/chat/completions") + # functions/function_call: null or empty functions[] are omit no-ops + # (SDK optional defaults); non-empty or any function_call fail closed. + functions_raw = body.get("functions") if "functions" in body else None + function_call_raw = body.get("function_call") if "function_call" in body else None + functions_present = ( + "functions" in body + and functions_raw is not None + and not (isinstance(functions_raw, list) and not functions_raw) + ) + # function_call none/auto/empty-string (whitespace-padded) without functions + # are omit-equivalent no-ops; any other function_call or non-empty functions + # fail closed. + function_call_present = ( + "function_call" in body + and function_call_raw is not None + and not ( + isinstance(function_call_raw, str) + and ( + not function_call_raw.strip() + or function_call_raw.strip() in ("none", "auto") + ) + ) + ) + if functions_present or function_call_present: + # OpenAI deprecated functions/function_call in favor of tools/tool_choice. + # Fail closed with a migration message rather than silent passthrough of + # a deprecated surface clients may still send from old SDKs. + raise RequestError( + 400, + "invalid_functions", + "functions and function_call are not supported on /v1/chat/completions; " + "use tools and tool_choice instead", + ) + tools_list = body.get("tools") if isinstance(body.get("tools"), list) else None + # tool_choice null is omit-equivalent; alone / empty tools: only "none" is a valid no-op. + if ( + "tool_choice" in body + and body.get("tool_choice") is not None + and not tools_list + ): + tc = body.get("tool_choice") + tc_norm = tc.strip() if isinstance(tc, str) else tc + # none/auto/empty-object/empty-string without tools are omit-equivalent no-ops. + if ( + tc_norm not in ("none", "auto") + and not (isinstance(tc, dict) and not tc) + and not (isinstance(tc, str) and not tc.strip()) + ): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice requires tools on /v1/chat/completions", + ) + # Shape-check tool results and message audio/function_call before + # passthrough or orchestration (named errors, not silent drop). + _validate_chat_message_known_fields(body) + _validate_chat_tool_message_ids(body) + _validate_chat_assistant_tool_calls(body) + _validate_chat_message_audio_function_call(body) + if "response_format" in body: + _validate_chat_response_format(body) + if "tools" in body: + _validate_chat_tools(body) + if "tool_choice" in body: + _validate_chat_tool_choice(body) + if "parallel_tool_calls" in body: + # Always type-check. With tools, true/false both valid for + # provider passthrough; without tools, true fails closed. + # Explicit JSON null is treat-as-omit (SDK optional default). + ptc = body.get("parallel_tool_calls") + # Empty/whitespace string is treat-as-omit (SDK optional default). + if isinstance(ptc, str) and not ptc.strip(): + ptc = None + if ptc is not None: + if not isinstance(ptc, bool): + raise RequestError( + 400, + "invalid_parallel_tool_calls", + "parallel_tool_calls must be a boolean", + ) + if ptc is True and not tools_list: + raise RequestError( + 400, + "invalid_parallel_tool_calls", + "parallel_tool_calls=true requires tools on /v1/chat/completions", + ) + # Explicit JSON null on trigger keys is omit-equivalent (SDK optional + # defaults) — do not force single-agent passthrough for null-only keys. + if any( + key in body and body.get(key) is not None + for key in PASSTHROUGH_TRIGGER_KEYS + ): # response_format / tools cannot be merged across agents; # proxy the full request to one agent and return it verbatim. started_at = time.perf_counter() @@ -733,36 +3697,226 @@ def do_POST(self) -> None: # noqa: N802 return messages = _validate_messages(body.get("messages")) mode = _validate_mode(body.get("orchestration") or body.get("orchestration_mode") or body.get("mode") or "auto") - include_trace = bool(body.get("include_orchestration_trace", security.expose_trace_by_default)) + if "include_orchestration_trace" in body: + include_trace_raw = body.get("include_orchestration_trace") + # Explicit JSON null is treat-as-omit (SDK optional default). + if include_trace_raw is None or ( + isinstance(include_trace_raw, str) and not include_trace_raw.strip() + ): + include_trace = bool(security.expose_trace_by_default) + elif not isinstance(include_trace_raw, bool): + raise RequestError( + 400, + "invalid_include_orchestration_trace", + "include_orchestration_trace must be a boolean", + ) + else: + include_trace = include_trace_raw + else: + include_trace = bool(security.expose_trace_by_default) stream = body.get("stream", False) + # Explicit JSON null or empty/whitespace string is treat-as-omit + # (SDK optional default → non-stream). + if stream is None or (isinstance(stream, str) and not stream.strip()): + stream = False if not isinstance(stream, bool): raise RequestError(400, "invalid_request", "stream must be a boolean") + if "stream_options" in body: + _validate_chat_stream_options(body, stream) attribution = _validate_attribution(body.get("attribution")) routing = _validate_routing(body.get("routing")) - model_name = str(body.get("model", "contextual-orchestrator")) + # Require model — silent default to contextual-orchestrator hid + # which deployment the buyer selected on the chat Completions path. + model_name = _validate_completions_model(body) + _require_pool_model(orchestrator, model_name) + attribution = dict(attribution or {}) + # OpenAI chat ``user`` → account when unset. + # Same fail-closed rules as Completions: present key must be a + # non-empty string ≤64 chars (null/empty/non-string rejected). + end_user_id = _validate_completions_user(body) + if end_user_id is not None and not attribution.get("account"): + attribution["account"] = end_user_id + if model_name and not attribution.get("model_name"): + attribution["model_name"] = model_name + if not attribution.get("service"): + attribution["service"] = "chat_completions_api" + temperature = None + top_p = None + max_tokens = None + presence_penalty = None + frequency_penalty = None + if "temperature" in body: + temperature = _validate_completions_temperature(body) + if "top_p" in body: + top_p = _validate_completions_top_p(body) + # OpenAI: max_completion_tokens takes precedence over max_tokens. + if "max_completion_tokens" in body: + max_tokens = _validate_chat_max_completion_tokens(body) + elif "max_tokens" in body: + max_tokens = _validate_completions_max_tokens(body) + if "presence_penalty" in body: + presence_penalty = _validate_completions_presence_penalty(body) + if "frequency_penalty" in body: + frequency_penalty = _validate_completions_frequency_penalty(body) + if "seed" in body: + # Type-check then fail closed: chat route does not apply seed. + # Explicit JSON null or empty/whitespace string is treat-as-omit. + seed_raw = body.get("seed") + if seed_raw is not None and not ( + isinstance(seed_raw, str) and not seed_raw.strip() + ): + try: + _validate_completions_seed(body) + except RequestError as exc: + if exc.code == "invalid_seed" and "not supported" in exc.message: + raise RequestError( + 400, + "invalid_seed", + "seed is not supported on /v1/chat/completions", + ) from exc + raise + raise RequestError( + 400, + "invalid_seed", + "seed is not supported on /v1/chat/completions", + ) + if "logit_bias" in body: + # Empty {} is an honest no-op (shared Completions helper). + # Non-empty maps fail closed with a chat-path message. + try: + _validate_completions_logit_bias(body) + except RequestError as exc: + if ( + exc.code == "invalid_logit_bias" + and "not supported" in exc.message + ): + raise RequestError( + 400, + "invalid_logit_bias", + "logit_bias is not supported on /v1/chat/completions", + ) from exc + raise + if "stop" in body: + # Explicit JSON null, empty string, empty [], or all-whitespace + # array items is treat-as-omit (SDK optional default). + stop_val = body.get("stop") + if isinstance(stop_val, list): + stop_val = [s for s in stop_val if not (isinstance(s, str) and not s.strip())] + if not stop_val: + stop_val = [] + if stop_val is not None and stop_val != [] and stop_val != "": + try: + _validate_completions_stop(body) + except RequestError as exc: + # Completions helper fails closed with a Completions path message; + # re-surface for chat with the chat endpoint string. + if exc.code == "invalid_stop" and "not supported" in exc.message: + raise RequestError( + 400, + "invalid_stop", + "stop sequences are not supported on /v1/chat/completions", + ) from exc + raise + raise RequestError( + 400, + "invalid_stop", + "stop sequences are not supported on /v1/chat/completions", + ) + if "n" in body: + try: + _validate_completions_n(body) + except RequestError as exc: + if exc.code == "invalid_n" and "not supported" in exc.message: + raise RequestError( + 400, + "invalid_n", + "n greater than 1 is not supported on /v1/chat/completions", + ) from exc + raise + if "logprobs" in body or "top_logprobs" in body: + # Chat route path does not return token logprobs; fail closed. + # Explicit JSON null is treat-as-omit (SDK optional default). + if "logprobs" in body: + lp = body.get("logprobs") + # Empty/whitespace string is treat-as-omit. + if isinstance(lp, str) and not lp.strip(): + lp = None + if lp is not None: + if not isinstance(lp, bool): + raise RequestError(400, "invalid_logprobs", "logprobs must be a boolean") + if lp is True: + raise RequestError( + 400, + "invalid_logprobs", + "logprobs=true is not supported on /v1/chat/completions", + ) + if "top_logprobs" in body: + # Explicit JSON null or 0 is treat-as-omit (SDK optional default). + tlp = body.get("top_logprobs") + if tlp is not None and tlp != 0: + raise RequestError( + 400, + "invalid_top_logprobs", + "top_logprobs is not supported on /v1/chat/completions", + ) + if "store" in body: + _validate_chat_store(body) + if "modalities" in body: + _validate_chat_modalities(body) + if "prediction" in body: + _validate_chat_prediction(body) + if "reasoning_effort" in body: + _validate_chat_reasoning_effort(body) + if "service_tier" in body: + _validate_service_tier(body, endpoint_path="/v1/chat/completions") + if "metadata" in body: + _validate_openai_metadata(body) started_at = time.perf_counter() - if stream and orchestrator.would_route(messages, mode): - self._stream_route_completion(orchestrator, security, messages, model_name) - orchestrator.record_analytics_event( - "chat_completion_requested", - { - "endpoint_path": "/v1/chat/completions", - "actor_scope": "inference", - "status_code": 200, - "run_mode": "route", - "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), - "response_streamed": True, - }, - ) - return - result = self._run(lambda: coordinator.complete( - messages, - mode=mode, - attribution=attribution, - hints=routing, - model_name=model_name, - workflow_run_id=f"run_{uuid.uuid4().hex}", - )) + model_client = orchestrator.client + previous_max_tokens = model_client.max_output_tokens + previous_temperature = model_client.default_temperature + previous_top_p = model_client.default_top_p + previous_presence = model_client.default_presence_penalty + previous_frequency = model_client.default_frequency_penalty + if max_tokens is not None: + model_client.max_output_tokens = max_tokens + if temperature is not None: + model_client.default_temperature = temperature + if top_p is not None: + model_client.default_top_p = top_p + if presence_penalty is not None: + model_client.default_presence_penalty = presence_penalty + if frequency_penalty is not None: + model_client.default_frequency_penalty = frequency_penalty + try: + if stream and orchestrator.would_route(messages, mode): + self._stream_route_completion(orchestrator, security, messages, model_name) + orchestrator.record_analytics_event( + "chat_completion_requested", + { + "endpoint_path": "/v1/chat/completions", + "actor_scope": "inference", + "status_code": 200, + "run_mode": "route", + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + "response_streamed": True, + }, + ) + return + result = self._run(lambda: coordinator.complete( + messages, + mode=mode, + attribution=attribution, + hints=routing, + model_name=model_name, + workflow_run_id=f"run_{uuid.uuid4().hex}", + )) + finally: + model_client.max_output_tokens = previous_max_tokens + model_client.default_temperature = previous_temperature + model_client.default_top_p = previous_top_p + model_client.default_presence_penalty = previous_presence + model_client.default_frequency_penalty = previous_frequency # Latency-tolerant requests get dispatched to the batch backend. if result.get("channel") == "batch": orchestrator.record_analytics_event( @@ -797,15 +3951,123 @@ def do_POST(self) -> None: # noqa: N802 result, model=model_name, include_trace=include_trace, usage=result.get("usage"), )) return + if path == "/v1/embeddings": + # OpenAI sync embeddings: input → vectors as list object. + # Reuses the embedding batch backend (local path completes + # synchronously) and frames an OpenAI-shaped response so + # SDKs that call /v1/embeddings work without the batch path. + _reject_unknown_keys(body, ALLOWED_EMBEDDINGS_KEYS) + model_name = _validate_embeddings_model(body) + # Same pool honesty as chat/Completions: do not silently serve + # a different embedding deployment than the client requested. + _require_pool_model(orchestrator, model_name) + _validate_embeddings_encoding_format(body) + _validate_embeddings_dimensions(body) + end_user_id = _validate_completions_user(body) + if "routing" in body: + routing = _validate_routing(body.get("routing")) + # Sync embeddings has no batch channel job plane. + if routing and routing.get("channel") == "batch": + raise RequestError( + 400, + "invalid_routing", + "routing.channel=batch is not supported on /v1/embeddings; use /v1/batch/embeddings", + ) + if routing and routing.get("latency_tolerant") is True: + raise RequestError( + 400, + "invalid_routing", + "routing.latency_tolerant=true is not supported on /v1/embeddings; use /v1/batch/embeddings", + ) + if "metadata" in body and not isinstance(body.get("metadata"), dict): + # OpenAI-shaped string metadata is preferred for this + # surface; non-objects fail closed before attribution merge. + raise RequestError(400, "invalid_metadata", "metadata must be an object") + if "metadata" in body: + # When all values are strings, enforce OpenAI ≤16 pairs; + # naruon-style attribution-in-metadata still uses + # _embeddings_attribution below for known dimensions. + meta = body.get("metadata") or {} + if meta and all(isinstance(v, str) for v in meta.values()): + _validate_openai_metadata(body) + if "input" not in body and "inputs" not in body: + # OpenAI only documents ``input``; accept nothing else. + raise RequestError(400, "invalid_input", "input is required on /v1/embeddings") + # Prefer OpenAI ``input``; do not accept ``inputs`` on this path + # (batch endpoint owns ``inputs``) so clients get a clear split. + if "inputs" in body and "input" not in body: + raise RequestError( + 400, + "invalid_input", + "use input on /v1/embeddings; inputs is only for /v1/batch/embeddings", + ) + inputs = _validate_embeddings_inputs({"input": body.get("input")}) + attribution = _embeddings_attribution(body) + attribution = dict(attribution or {}) + if end_user_id is not None and not attribution.get("account"): + attribution["account"] = end_user_id + if model_name and not attribution.get("model_name"): + attribution["model_name"] = model_name + if not attribution.get("service"): + attribution["service"] = "embeddings_api" + started_at = time.perf_counter() + document = self._run(lambda: coordinator.complete_embeddings_batch( + inputs, + model=model_name, + attribution=attribution, + metadata={"actor_scope": "inference", "endpoint_alias": "embeddings"}, + )) + if document.get("status") != "completed" or document.get("embeddings") is None: + # Async backends return a job handle; fail closed on the + # sync OpenAI path rather than inventing vectors. + raise RequestError( + 503, + "embeddings_unavailable", + "sync /v1/embeddings is unavailable for this backend; use /v1/batch/embeddings", + ) + orchestrator.record_analytics_event( + "embeddings_requested", + { + "endpoint_path": "/v1/embeddings", + "actor_scope": "inference", + "status_code": 200, + "input_count": len(inputs), + "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), + }, + ) + self._send(_openai_embeddings_response(document, model=model_name)) + return if path == "/v1/batch/embeddings": _reject_unknown_keys(body, ALLOWED_EMBEDDINGS_BATCH_KEYS) inputs = _validate_embeddings_inputs(body) - model_name = str(body.get("model", "contextual-orchestrator")) + # Require model — silent default to contextual-orchestrator was an + # honesty gap for naruon/batch clients that omit the field. + if "model" not in body: + raise RequestError( + 400, + "invalid_model", + "model is required on /v1/batch/embeddings", + ) + model_name = _validate_embeddings_model(body) + _require_pool_model(orchestrator, model_name) + _validate_embeddings_encoding_format(body) + _validate_embeddings_dimensions(body) + # OpenAI ``user`` end-user id — same fail-closed shape as sync embeddings. + end_user_id = _validate_completions_user(body) + if "routing" in body: + _validate_routing(body.get("routing")) attribution = _embeddings_attribution(body) + attribution = dict(attribution or {}) + if end_user_id is not None and not attribution.get("account"): + attribution["account"] = end_user_id + if model_name and not attribution.get("model_name"): + attribution["model_name"] = model_name + if not attribution.get("service"): + attribution["service"] = "embeddings_batch_api" submit_metadata: dict[str, Any] = {"actor_scope": "inference"} - endpoint_alias = body.get("endpoint") - if endpoint_alias: - submit_metadata["endpoint_alias"] = str(endpoint_alias) + endpoint_alias = _validate_batch_embeddings_endpoint(body) + if endpoint_alias is not None: + submit_metadata["endpoint_alias"] = endpoint_alias document = self._run(lambda: coordinator.complete_embeddings_batch( inputs, model=model_name, @@ -862,6 +4124,167 @@ def do_POST(self) -> None: # noqa: N802 # The Responses API has no chat-completions verifier equivalent, # so every request is proxied to one agent verbatim. _reject_unknown_keys(body, ALLOWED_RESPONSES_KEYS) + # Fail-closed shape checks before passthrough so buyers never + # get a 200 after shipping invalid OpenAI-shaped metadata/input. + _validate_responses_model(body) + _validate_responses_conversation_controls(body) + if "store" in body: + _validate_responses_store(body) + # OpenAI ``user`` end-user id — same fail-closed shape as chat/Completions. + if "user" in body: + _validate_completions_user(body) + if "service_tier" in body: + _validate_service_tier(body, endpoint_path="/v1/responses") + if "stream_options" in body: + _validate_responses_stream_options(body) + # Sampling knobs: type/range fail-closed before provider passthrough. + if "temperature" in body: + _validate_completions_temperature(body) + if "top_p" in body: + _validate_completions_top_p(body) + if "presence_penalty" in body: + _validate_completions_presence_penalty(body) + if "frequency_penalty" in body: + _validate_completions_frequency_penalty(body) + if "n" in body: + _validate_responses_n(body) + if "seed" in body: + _validate_responses_seed(body) + if "stop" in body: + _validate_responses_stop(body) + if "logit_bias" in body: + _validate_responses_logit_bias(body) + if "logprobs" in body or "top_logprobs" in body: + _validate_responses_logprobs(body) + if "max_tokens" in body: + _validate_completions_max_tokens(body) + if "max_completion_tokens" in body: + _validate_chat_max_completion_tokens(body) + if "max_output_tokens" in body: + _validate_responses_max_output_tokens(body) + if "max_tool_calls" in body: + _validate_responses_max_tool_calls(body) + _validate_openai_sdk_control_fields(body, endpoint_path="/v1/responses") + _validate_openai_background(body, endpoint_path="/v1/responses") + if "parallel_tool_calls" in body: + _validate_responses_parallel_tool_calls(body) + # Tools surface: same OpenAI function-tool shape as chat; fail closed. + functions_raw = body.get("functions") if "functions" in body else None + function_call_raw = body.get("function_call") if "function_call" in body else None + functions_present = ( + "functions" in body + and functions_raw is not None + and not (isinstance(functions_raw, list) and not functions_raw) + ) + # function_call none/auto/empty-string (whitespace-padded) without functions + # are omit-equivalent no-ops. + function_call_present = ( + "function_call" in body + and function_call_raw is not None + and not ( + isinstance(function_call_raw, str) + and ( + not function_call_raw.strip() + or function_call_raw.strip() in ("none", "auto") + ) + ) + ) + if functions_present or function_call_present: + raise RequestError( + 400, + "invalid_functions", + "functions and function_call are not supported on /v1/responses; " + "use tools and tool_choice instead", + ) + tools_list = body.get("tools") if isinstance(body.get("tools"), list) else None + # tool_choice null is omit-equivalent; alone / empty tools: only "none" is a valid no-op. + if ( + "tool_choice" in body + and body.get("tool_choice") is not None + and not tools_list + ): + tc = body.get("tool_choice") + tc_norm = tc.strip() if isinstance(tc, str) else tc + # none/auto/empty-object/empty-string without tools are omit-equivalent no-ops. + if ( + tc_norm not in ("none", "auto") + and not (isinstance(tc, dict) and not tc) + and not (isinstance(tc, str) and not tc.strip()) + ): + raise RequestError( + 400, + "invalid_tool_choice", + "tool_choice requires tools on /v1/responses", + ) + if "tools" in body: + _validate_chat_tools(body) + if "tool_choice" in body: + _validate_chat_tool_choice(body) + if "response_format" in body: + _validate_chat_response_format(body) + if "modalities" in body: + _validate_responses_modalities(body) + if "prediction" in body: + _validate_responses_prediction(body) + if "reasoning_effort" in body and body.get("reasoning_effort") is not None: + raise RequestError( + 400, + "invalid_reasoning_effort", + "reasoning_effort is not supported on /v1/responses", + ) + if "reasoning" in body: + _validate_responses_reasoning(body) + if "instructions" in body: + _validate_responses_instructions(body) + if "metadata" in body: + _validate_openai_metadata(body) + if "attribution" in body: + _validate_attribution(body.get("attribution")) + if "routing" in body: + routing = _validate_routing(body.get("routing")) + # Responses passthrough has no batch channel plane yet. + if routing and routing.get("channel") == "batch": + raise RequestError( + 400, + "invalid_routing", + "routing.channel=batch is not supported on /v1/responses", + ) + if routing and routing.get("latency_tolerant") is True: + raise RequestError( + 400, + "invalid_routing", + "routing.latency_tolerant=true is not supported on /v1/responses", + ) + if "input" not in body: + raise RequestError(400, "invalid_input", "input is required on /v1/responses") + input_value = body.get("input") + if not isinstance(input_value, (str, list)) or ( + isinstance(input_value, str) and not input_value.strip() + ) or (isinstance(input_value, list) and len(input_value) == 0): + raise RequestError( + 400, + "invalid_input", + "input must be a non-empty string or non-empty array on /v1/responses", + ) + # stream=false / omit → non-SSE JSON response (honest no-stream path). + # stream=true is not implemented for Responses passthrough. + if "stream" in body: + stream = body.get("stream") + # Explicit JSON null / false / empty string are omit-equivalent no-ops. + if ( + stream is None + or stream is False + or (isinstance(stream, str) and not stream.strip()) + ): + pass + elif not isinstance(stream, bool): + raise RequestError(400, "invalid_stream", "stream must be a boolean") + elif stream is True: + raise RequestError( + 400, + "invalid_stream", + "stream is not supported on /v1/responses", + ) started_at = time.perf_counter() proxied = self._run( lambda: orchestrator.proxy_completion(body, endpoint="responses") diff --git a/docs/architecture.md b/docs/architecture.md index c0f63a81e..f9d9c7255 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -37,6 +37,11 @@ This repository implements the interface and control plane, not the trained coor - `WorkflowStep.access`: Conductor-style visibility control. - `ModelClient`: OpenAI-compatible HTTP client, with `mock://` for local checks. - `contextual_orchestrator.server`: small `/v1/chat/completions` HTTP server. + Compatibility honesty: JSON-null SDK optionals on `tools[].function` and + `response_format.json_schema` are popped in place before + `proxy_completion` so providers see omit, not `null`. + `json_schema.name` and `tool.function.name` must match + `[a-zA-Z0-9_-]{1,64}` (ASCII only; `str.isalnum()` is not sufficient). The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses deterministic keyword scoring so the repo runs without training data, GPUs, or vendor credentials. diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 9897b2bd2..991bb5721 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -19,9 +19,13 @@ The surfaces were located with CodeGraph (`codegraph explore "parse decode deserialize request config validate untrusted input"`): 1. **HTTP request body** — `server._coerce_json` / `_reject_unknown_keys` / - `_validate_mode` / `_validate_messages`. Arbitrary bytes must normalise to a - validated structure or raise `RequestError` / a JSON decode error — never an - unhandled crash. + `_validate_mode` / `_validate_messages` / `_validate_chat_response_format` / + `_validate_chat_tools`. + Arbitrary bytes must normalise to a validated structure or raise + `RequestError` / a JSON decode error — never an unhandled crash. + Successful `json_schema` names and `tool.function.name` values match + `[a-zA-Z0-9_-]{1,64}` (ASCII `isascii()` plus alnum/`_`/`-`; Unicode + letters and digits fail closed). 2. **Agent config** — `orchestrator.ModelAgent.from_dict`. Arbitrary decoded JSON must yield a well-typed `ModelAgent` or raise `KeyError`/`TypeError`/ `ValueError`. diff --git a/docs/library_research.md b/docs/library_research.md index 42c7fa95c..4464f6dfd 100644 --- a/docs/library_research.md +++ b/docs/library_research.md @@ -53,6 +53,19 @@ Extraction triggers: Until those triggers exist, Ponytail recommends strengthening the current single-repo product instead of splitting it. +## Structured-output optional-field honesty (2026-08-16) + +Researched before adding `_validate_chat_response_format` inner-field omit: + +| Option | Decision | Why | +|---|---|---| +| Official OpenAI Chat Completions + Structured Outputs docs | Cite (APA 7th) in `docs/papers/README.md`; do not vendor PDFs | Authoritative field list: `name`, `description`, `schema`, `strict`; `json_schema.name` and `tool.function.name` charset `[a-zA-Z0-9_-]{1,64}` (ASCII; `str.isalnum()` is not the regex) | +| IETF JSON Schema 2020-12 (Wright et al., 2022) | Cite; type-check object only | Full schema evaluation is a provider concern; this gateway only guarantees omit-real optionals and fail-closed unknown keys | +| Pydantic / jsonschema library | Skip | Stdlib `dict.pop` + `isinstance` covers the honesty contract; Ponytail forbids a new runtime dependency | +| Silent drop of unknown `json_schema` keys | Skip | Buyers cannot debug smuggled fields; named `invalid_response_format` is the next action | + +No new dependency. Implementation stays in `contextual_orchestrator/server.py`. + ## Required For New Designs Every new subsystem design must update this file before implementation starts. The entry must name the existing libraries researched, the selected library or stdlib alternative, and the custom code that was deliberately skipped. diff --git a/docs/papers/README.md b/docs/papers/README.md index 65a89d2af..1f4427d72 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -35,6 +35,35 @@ redistribution; each is cited below with its arXiv identifier. the responsive path. Distributed under the arXiv non-exclusive license / CC BY as marked on arXiv. +## API-shape honesty (structured outputs) + +- OpenAI. (2024). *Create chat completion*. OpenAI API reference. + https://platform.openai.com/docs/api-reference/chat/create + Grounds the official `tools[].function` and `response_format.json_schema` + optional fields (`description`, `parameters`/`schema`, `strict`) and the + `tools[].function.name` charset `[a-zA-Z0-9_-]{1,64}` (ASCII only; + `str.isalnum()` is not the regex). Official SDKs serialize omitted + optionals as JSON `null`; this gateway pops those keys before proxy so + passthrough matches omit. Copyrighted vendor docs — cite + link only; + no PDF vendored. + +- OpenAI. (2024). *Structured outputs*. OpenAI Platform. + https://platform.openai.com/docs/guides/structured-outputs + Grounds fail-closed unknown keys on the `json_schema` object, the + requirement that `schema` is a JSON Schema object when structured output + is requested, and the `name` charset `[a-zA-Z0-9_-]{1,64}` (ASCII only; + `str.isalnum()` is not the regex). Copyrighted vendor docs — cite + + link only. + +- Wright, A., Andrews, H., Hutton, B., & Dennis, G. (2022). *JSON Schema: A + media type for describing JSON documents* (Internet-Draft + draft-bhutton-json-schema-01). Internet Engineering Task Force. + https://datatracker.ietf.org/doc/html/draft-bhutton-json-schema-01 + Latest widely deployed JSON Schema dialect (2020-12) used when this + gateway type-checks `tool.function.parameters` and + `response_format.json_schema.schema` as objects without re-implementing + full schema validation. IETF Internet-Draft; cite + link only. + ## Batch execution / load balancing The external `pg-llm-batch` service carries its own grounding papers, including diff --git a/docs/rest_api_design.md b/docs/rest_api_design.md index 9378e5a37..8c87ff83b 100644 --- a/docs/rest_api_design.md +++ b/docs/rest_api_design.md @@ -93,6 +93,15 @@ These product surfaces are now implemented in this prototype: | `GET` | `/api/v1/commercial_due_diligence_rooms/latest` | Produce the buyer due diligence room that ties purchase approval, runtime API evidence, admin trace/access evidence, security, commercial terms, value analytics, implementation readiness, Figma, review-process policy, packaging decision, and buyer/external missing artifacts into one runtime diligence artifact. | Fugu API adoption; TRINITY verification; Conductor trace/access evidence; buyer diligence committee review. | | `GET` | `/api/v1/commercial_investment_committee_memos/latest` | Produce the investment committee memo that ties due diligence, purchase approval, financial case, risk/security, commercial terms, implementation readiness, Figma, review-process policy, packaging decision, and buyer/external approval conditions into one executive recommendation artifact. | Fugu API adoption; TRINITY verification; Conductor trace/access evidence; executive investment committee review. | +## Compatibility honesty (caller next action) + +When an official OpenAI SDK serializes an omitted optional as JSON `null` or a blank string, do **not** send that key to a provider yourself and do **not** assume this gateway will forward it. On `/v1/chat/completions` and `/v1/responses`: + +- `tools[].function.description` / `parameters` / `strict` JSON `null` are popped before proxy so the upstream payload matches omit. Empty or whitespace-only `description` is also popped. `tools[].function.name` must match `[a-zA-Z0-9_-]{1,64}` (ASCII letters, digits, underscore, hyphen — Unicode letters and digits are rejected) or the gateway returns `invalid_tools` — rename the function to the ASCII charset, then retry. +- `response_format.json_schema` accepts only `name`, `description`, `schema`, and `strict`. `name` must match `[a-zA-Z0-9_-]{1,64}` (ASCII letters, digits, underscore, hyphen — Unicode letters and digits are rejected). JSON-null or blank `description` and JSON-null `strict` are popped in place. Unknown inner keys, illegal names, and non-string `description` return `invalid_response_format` — rename the schema to the ASCII charset, then retry. + +If a provider still rejects the request, inspect `echo.response_format` / `echo.tools` on the mock path first; a remaining null key is a gateway bug, not a provider quirk. + ## Production Library Target FastAPI should replace the current stdlib HTTP adapter when the API needs authentication, richer OpenAPI schema generation, dependency injection, and typed request/response models. diff --git a/fuzz/targets.py b/fuzz/targets.py index d0c344462..dbac0fdf5 100644 --- a/fuzz/targets.py +++ b/fuzz/targets.py @@ -12,7 +12,8 @@ consume untrusted bytes/JSON: 1. ``server._coerce_json`` / ``_validate_mode`` / ``_validate_messages`` / - ``_reject_unknown_keys`` -- the HTTP request-body parser and validators. + ``_reject_unknown_keys`` / ``_validate_chat_response_format`` -- the HTTP + request-body parser and validators. 2. ``orchestrator.ModelAgent.from_dict`` -- the agent-pool config parser. 3. ``orchestrator.redact_text`` / ``redact_value`` -- secret/PII redaction run over arbitrary trace payloads (regex + recursion). @@ -106,6 +107,46 @@ def exercise_request_body(raw: bytes) -> None: assert message["role"] in server.ALLOWED_MESSAGE_ROLES assert isinstance(message["content"], str) + # Structured-output honesty: omit-real optionals + fail-closed name charset. + if "response_format" in body: + try: + fmt = server._validate_chat_response_format(body) + except RequestError: + pass + else: + if fmt is not None: + assert isinstance(fmt, dict) + if fmt.get("type") == "json_schema": + schema = fmt.get("json_schema") + assert isinstance(schema, dict) + name = schema.get("name") + assert isinstance(name, str) and 1 <= len(name) <= 64 + assert name.isascii() and all( + ch.isalnum() or ch in "_-" for ch in name + ) + description = schema.get("description") + if description is not None: + assert isinstance(description, str) and description.strip() + if "strict" in schema: + assert isinstance(schema["strict"], bool) + + # Tools honesty: successful function names match [a-zA-Z0-9_-]{1,64} ASCII. + if "tools" in body: + try: + tools = server._validate_chat_tools(body) + except RequestError: + pass + else: + if tools: + for item in tools: + function = item.get("function") + assert isinstance(function, dict) + tool_name = function.get("name") + assert isinstance(tool_name, str) and 1 <= len(tool_name) <= 64 + assert tool_name.isascii() and all( + ch.isalnum() or ch in "_-" for ch in tool_name + ) + def exercise_agent_config(value: Any) -> None: """Drive ``ModelAgent.from_dict`` over an arbitrary decoded JSON value.""" diff --git a/tests/fuzz/test_fuzz_properties.py b/tests/fuzz/test_fuzz_properties.py index 7e7b3f347..05f44c33f 100644 --- a/tests/fuzz/test_fuzz_properties.py +++ b/tests/fuzz/test_fuzz_properties.py @@ -73,6 +73,32 @@ def test_request_body_rejects_unhashable_message_role() -> None: exercise_request_body(b'{"messages":[{"role":[],"":[],"modnt":""}]}') +@_SETTINGS +@given( + st.fixed_dictionaries( + { + "response_format": st.fixed_dictionaries( + { + "type": st.just("json_schema"), + "json_schema": st.fixed_dictionaries( + { + "name": st.text(max_size=80), + "schema": st.just({"type": "object", "properties": {}}), + }, + optional={ + "description": st.none() | st.text(max_size=40), + "strict": st.none() | st.booleans() | st.integers(), + }, + ), + } + ) + } + ).map(lambda v: json.dumps(v).encode("utf-8")) +) +def test_request_body_json_schema_name_never_crashes(raw: bytes) -> None: + exercise_request_body(raw) + + @_SETTINGS @given(_json_values) def test_agent_config_parser(value: object) -> None: diff --git a/tests/test_analytics_runtime.py b/tests/test_analytics_runtime.py index 5fad97af6..83c77888f 100644 --- a/tests/test_analytics_runtime.py +++ b/tests/test_analytics_runtime.py @@ -114,7 +114,7 @@ def test_analytics_endpoint_and_admin_console_use_source_backed_snapshot() -> No try: chat_status, _ = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "hello"}]}, + {"model": "mock-planner", "messages": [{"role": "user", "content": "hello"}]}, "secret_token", ) snapshot_status, snapshot = get_json( diff --git a/tests/test_assistant_refusal_annotations_http_honesty.py b/tests/test_assistant_refusal_annotations_http_honesty.py new file mode 100644 index 000000000..4990ab32c --- /dev/null +++ b/tests/test_assistant_refusal_annotations_http_honesty.py @@ -0,0 +1,144 @@ +"""Assistant refusal/annotations null-empty omit and non-empty fail-closed honesty.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "assistant_refusal_annotations_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_refusal_null_and_empty() -> None: + server, thread, port = _server() + try: + for refusal in (None, "", " "): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": "prior", "refusal": refusal}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 200, (refusal, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_nonempty_refusal() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": None, "refusal": "I cannot help"}, + {"role": "user", "content": "hi"}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_refusal" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_annotations_null_and_empty() -> None: + server, thread, port = _server() + try: + for annotations in (None, []): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": "prior", "annotations": annotations}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 200, (annotations, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_nonempty_annotations() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "assistant", + "content": "prior", + "annotations": [{"type": "url_citation", "url": "https://x"}], + }, + {"role": "user", "content": "hi"}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_annotations" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_refusal_null_and_empty() + test_http_chat_rejects_nonempty_refusal() + test_http_chat_accepts_annotations_null_and_empty() + test_http_chat_rejects_nonempty_annotations() + print("ok") diff --git a/tests/test_assistant_tool_calls_null_noop_http_honesty.py b/tests/test_assistant_tool_calls_null_noop_http_honesty.py new file mode 100644 index 000000000..740e832bf --- /dev/null +++ b/tests/test_assistant_tool_calls_null_noop_http_honesty.py @@ -0,0 +1,163 @@ +"""Assistant tool_calls null/empty and content null honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "assistant_tool_calls_null_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_assistant_tool_calls_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "ok", "tool_calls": None}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_assistant_content_null_with_tool_calls() -> None: + """OpenAI tool turns often send content:null with tool_calls.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "call tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + {"role": "user", "content": "thanks"}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_tool_calls_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "ok", "tool_calls": []}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_user_content_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": None}], + }, + ) + assert status == 400, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_malformed_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"type": "function"}], + }, + ], + }, + ) + assert status == 400, body + assert "invalid_message" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_audio_websearch_reasoning_null_noop_http_honesty.py b/tests/test_audio_websearch_reasoning_null_noop_http_honesty.py new file mode 100644 index 000000000..6268e44de --- /dev/null +++ b/tests/test_audio_websearch_reasoning_null_noop_http_honesty.py @@ -0,0 +1,164 @@ +"""audio/web_search_options/reasoning null as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "audio_websearch_reasoning_null_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_null_audio_web_search_reasoning() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null audio websearch reasoning"}], + "audio": None, + "web_search_options": None, + "reasoning": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_audio_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "audio obj"}], + "audio": {"voice": "alloy", "format": "mp3"}, + }, + ) + assert status == 400, body + assert "invalid_audio" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_web_search_options_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "websearch"}], + "web_search_options": {"search_context_size": "medium"}, + }, + ) + assert status == 400, body + assert "invalid_web_search_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_reasoning_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "reasoning obj"}], + "reasoning": {"effort": "medium"}, + }, + ) + assert status == 400, body + assert "invalid_reasoning" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_null_chat_era_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "null chat era", + "modalities": None, + "prediction": None, + "reasoning_effort": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_nonnull_chat_era_field() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "modalities audio", + "modalities": ["audio"], + }, + ) + assert status == 400, body + assert "invalid_chat_era_field" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_background_reasoning_reject_http_honesty.py b/tests/test_background_reasoning_reject_http_honesty.py new file mode 100644 index 000000000..04e854764 --- /dev/null +++ b/tests/test_background_reasoning_reject_http_honesty.py @@ -0,0 +1,164 @@ +"""OpenAI background mode and Completions reasoning object reject honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "background_reasoning_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_background_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "background"}], + "background": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_background" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_background_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "background responses", + "background": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_background" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_reasoning_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "reasoning object", + "reasoning": {"effort": "high"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_reasoning" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_baseline_without_background() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "baseline"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_background_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "background false"}], + "background": False, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_background_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "background false responses", + "background": False, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_batch_embeddings.py b/tests/test_batch_embeddings.py index 3d5e02fb5..07d2992a5 100644 --- a/tests/test_batch_embeddings.py +++ b/tests/test_batch_embeddings.py @@ -56,7 +56,15 @@ def _serve(): provider_name="mock", tags=("reasoning", "coding", "writing"), priority=1, - ) + ), + ModelAgent( + id="embedding_worker", + model="text-embedding-test", + base_url="mock://embed", + provider_name="acme-provider", + tags=("embedding", "offline_test"), + priority=2, + ), ] orchestrator = TaskOrchestrator(agents) config = InMemoryConfigStore() diff --git a/tests/test_batch_embeddings_encoding_dimensions_http_honesty.py b/tests/test_batch_embeddings_encoding_dimensions_http_honesty.py new file mode 100644 index 000000000..c1317dbcd --- /dev/null +++ b/tests/test_batch_embeddings_encoding_dimensions_http_honesty.py @@ -0,0 +1,120 @@ +"""Batch embeddings encoding_format and dimensions honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "batch_embeddings_encoding_dimensions_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/batch/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_batch_embeddings_accepts_encoding_format_float() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["float ok"], + "encoding_format": "float", + }, + ) + assert status in (200, 202), body + assert "unknown_fields" not in json.dumps(body) + assert "invalid_encoding_format" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_encoding_format_base64() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["base64"], + "encoding_format": "base64", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_encoding_format" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_dimensions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["dims"], + "dimensions": 128, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_dimensions" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_omit_encoding_and_dimensions_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "inputs": ["plain"]}, + ) + assert status in (200, 202), body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_batch_embeddings_endpoint_http_honesty.py b/tests/test_batch_embeddings_endpoint_http_honesty.py new file mode 100644 index 000000000..41bbad5aa --- /dev/null +++ b/tests/test_batch_embeddings_endpoint_http_honesty.py @@ -0,0 +1,160 @@ +"""Batch embeddings endpoint alias honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "batch_embeddings_endpoint_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/batch/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_batch_embeddings_accepts_omitted_endpoint() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "inputs": ["alpha chunk", "beta chunk"]}, + ) + assert status == 200, body + assert body.get("status") == "completed" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_non_empty_endpoint_alias() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["alpha chunk"], + "endpoint": "/v1/embeddings", + }, + ) + assert status == 200, body + assert body.get("status") == "completed" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_empty_endpoint_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["alpha chunk"], + "endpoint": " ", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_null_endpoint_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["alpha chunk"], + "endpoint": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_non_string_endpoint() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["alpha chunk"], + "endpoint": 123, + }, + ) + assert status == 400, body + assert "endpoint" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_overlong_endpoint() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["alpha chunk"], + "endpoint": "x" * 257, + }, + ) + assert status == 400, body + assert "endpoint" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_batch_embeddings_accepts_omitted_endpoint() + test_http_batch_embeddings_accepts_non_empty_endpoint_alias() + test_http_batch_embeddings_accepts_empty_endpoint_as_omit() + test_http_batch_embeddings_accepts_null_endpoint_as_omit() + test_http_batch_embeddings_rejects_non_string_endpoint() + test_http_batch_embeddings_rejects_overlong_endpoint() + print("ok") diff --git a/tests/test_batch_embeddings_routing_http_honesty.py b/tests/test_batch_embeddings_routing_http_honesty.py new file mode 100644 index 000000000..ae041449a --- /dev/null +++ b/tests/test_batch_embeddings_routing_http_honesty.py @@ -0,0 +1,135 @@ +"""Batch embeddings routing shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "batch_embeddings_routing_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/batch/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_batch_embeddings_accepts_priority_bulk() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["bulk priority"], + "routing": {"priority": "bulk"}, + }, + ) + assert status in (200, 202), body + assert "unknown_fields" not in json.dumps(body) + assert "invalid_routing" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_channel_batch() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["channel batch"], + "routing": {"channel": "batch", "latency_tolerant": True}, + }, + ) + assert status in (200, 202), body + assert "unknown_fields" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_invalid_priority() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["bad priority"], + "routing": {"priority": "urgent"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_routing" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_channel_turbo() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["turbo"], + "routing": {"channel": "turbo"}, + }, + ) + assert status == 400, body + assert "invalid_routing" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_baseline_without_routing() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "inputs": ["baseline"]} + ) + assert status in (200, 202), body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_batch_embeddings_user_http_honesty.py b/tests/test_batch_embeddings_user_http_honesty.py new file mode 100644 index 000000000..92e475bc8 --- /dev/null +++ b/tests/test_batch_embeddings_user_http_honesty.py @@ -0,0 +1,135 @@ +"""Batch embeddings OpenAI user field honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "batch_embeddings_user_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/batch/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_batch_embeddings_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["batch with user"], + "user": "end-user-batch-1", + }, + ) + assert status in (200, 202), body + blob = json.dumps(body) + assert "unknown_fields" not in blob + assert "invalid_user" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_omit_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "inputs": ["batch no user"]}, + ) + assert status in (200, 202), body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["batch empty user"], + "user": " ", + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + assert "unknown_fields" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_null_user_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["batch null user"], + "user": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_user_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "inputs": ["batch long user"], + "user": "u" * 65, + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_budget_enforcement.py b/tests/test_budget_enforcement.py index 6ec0816b4..3f3003a8c 100644 --- a/tests/test_budget_enforcement.py +++ b/tests/test_budget_enforcement.py @@ -86,7 +86,7 @@ def test_http_over_budget_returns_429() -> None: port = server.server_address[1] request = urllib.request.Request( f"http://127.0.0.1:{port}/v1/chat/completions", - data=json.dumps({"messages": [{"role": "user", "content": "blocked"}]}).encode("utf-8"), + data=json.dumps({"model": "test-model", "messages": [{"role": "user", "content": "blocked"}]}).encode("utf-8"), headers={"content-type": "application/json", "authorization": f"Bearer {token}", "connection": "close"}, method="POST", ) diff --git a/tests/test_chat_assistant_tool_calls_http_honesty.py b/tests/test_chat_assistant_tool_calls_http_honesty.py new file mode 100644 index 000000000..b25a0cc64 --- /dev/null +++ b/tests/test_chat_assistant_tool_calls_http_honesty.py @@ -0,0 +1,269 @@ +"""Chat Completions assistant tool_calls array shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_assistant_tool_calls_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _valid_tool_call(*, call_id: str = "call_1", name: str = "lookup_item") -> dict: + return { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": "{\"q\":\"x\"}"}, + } + + +def test_http_chat_accepts_assistant_tool_calls_shape() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "use the tool"}, + { + "role": "assistant", + "content": "", + "tool_calls": [_valid_tool_call()], + }, + { + "role": "tool", + "content": "result payload", + "tool_call_id": "call_1", + }, + {"role": "user", "content": "thanks"}, + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_calls_on_user_message() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": "nope", + "tool_calls": [_valid_tool_call()], + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob + assert "assistant" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_tool_calls_array_as_omit() -> None: + """Empty tool_calls is omit-equivalent (SDK no-op history slot).""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "", "tool_calls": []}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_calls_missing_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "type": "function", + "function": {"name": "lookup_item", "arguments": "{}"}, + } + ], + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob + assert "id" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_calls_bad_type() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "custom", + "function": {"name": "lookup_item", "arguments": "{}"}, + } + ], + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob + assert "function" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_calls_non_string_arguments() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup_item", "arguments": {"q": 1}}, + } + ], + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob + assert "arguments" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_calls_bad_function_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "bad name!", "arguments": "{}"}, + } + ], + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob + assert "function.name" in blob or "a-zA-Z0-9" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_assistant_tool_calls_shape() + test_http_chat_rejects_tool_calls_on_user_message() + test_http_chat_accepts_empty_tool_calls_array_as_omit() + test_http_chat_rejects_tool_calls_missing_id() + test_http_chat_rejects_tool_calls_bad_type() + test_http_chat_rejects_tool_calls_non_string_arguments() + test_http_chat_rejects_tool_calls_bad_function_name() + print("ok") diff --git a/tests/test_chat_attribution_routing_http_honesty.py b/tests/test_chat_attribution_routing_http_honesty.py new file mode 100644 index 000000000..74890ad2a --- /dev/null +++ b/tests/test_chat_attribution_routing_http_honesty.py @@ -0,0 +1,211 @@ +"""Chat Completions attribution and routing shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_attribution_routing_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_known_attribution_and_routing() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "known dims"}], + "attribution": {"team": "platform", "company": "acme"}, + "routing": {"channel": "sync", "priority": "interactive"}, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_unknown_attribution_dimension() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad dim"}], + "attribution": {"team": "platform", "cost_center": "xyz"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_attribution" in blob + assert "unsupported" in blob or "cost_center" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_attribution_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "attr string"}], + "attribution": "team=platform", + }, + ) + assert status == 400, body + assert "invalid_attribution" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_routing_unknown_key() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "routing junk"}], + "routing": {"channel": "sync", "region": "us-east"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_routing" in blob + assert "unsupported" in blob or "region" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_routing_latency_tolerant_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "latency string"}], + "routing": {"latency_tolerant": "yes"}, + }, + ) + assert status == 400, body + assert "invalid_routing" in json.dumps(body) + assert "boolean" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_routing_invalid_channel() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "channel bad"}], + "routing": {"channel": "turbo"}, + }, + ) + assert status == 400, body + assert "invalid_routing" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_routing_invalid_priority() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "priority bad"}], + "routing": {"priority": "urgent"}, + }, + ) + assert status == 400, body + assert "invalid_routing" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_batch_routing_latency_tolerant() -> None: + """latency_tolerant true should select batch channel (202 job handle).""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "batch me"}], + "routing": {"latency_tolerant": True}, + }, + ) + assert status == 202, body + assert body.get("channel") == "batch" or "job_id" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_known_attribution_and_routing() + test_http_chat_rejects_unknown_attribution_dimension() + test_http_chat_rejects_attribution_non_object() + test_http_chat_rejects_routing_unknown_key() + test_http_chat_rejects_routing_latency_tolerant_non_boolean() + test_http_chat_rejects_routing_invalid_channel() + test_http_chat_rejects_routing_invalid_priority() + test_http_chat_accepts_batch_routing_latency_tolerant() + print("ok") diff --git a/tests/test_chat_audio_web_search_reject_http_honesty.py b/tests/test_chat_audio_web_search_reject_http_honesty.py new file mode 100644 index 000000000..d34f1a1b8 --- /dev/null +++ b/tests/test_chat_audio_web_search_reject_http_honesty.py @@ -0,0 +1,105 @@ +"""Chat audio and web_search_options reject honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_audio_web_search_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_baseline_without_audio_or_web_search() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello baseline"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_audio() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello audio"}], + "audio": {"voice": "alloy", "format": "wav"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_audio" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_web_search_options() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello web search"}], + "web_search_options": {"search_context_size": "medium"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_web_search_options" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_chat_developer_multimodal_content_http_honesty.py b/tests/test_chat_developer_multimodal_content_http_honesty.py new file mode 100644 index 000000000..319fb0793 --- /dev/null +++ b/tests/test_chat_developer_multimodal_content_http_honesty.py @@ -0,0 +1,170 @@ +"""Chat message content honesty: developer role and multimodal arrays fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_message_content_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_string_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "plain text invoice note"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_developer_role() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "developer", "content": "system-like instructions"}, + {"role": "user", "content": "hi"}, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_role" in blob + assert "developer" in blob + assert "system" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_multipart_image_content() -> None: + """Vision callers send text+image_url parts; shape-check and passthrough.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this receipt"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/receipt.png"}, + }, + ], + } + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_input_audio_content_part() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": {"data": "AAAA", "format": "wav"}, + } + ], + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_non_string_non_array_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": 12345}], + }, + ) + assert status == 400, body + assert "invalid_message_content" in json.dumps(body) or "invalid_message" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_string_content() + test_http_chat_rejects_developer_role() + test_http_chat_accepts_multipart_image_content() + test_http_chat_rejects_input_audio_content_part() + test_http_chat_rejects_non_string_non_array_content() + print("ok") diff --git a/tests/test_chat_empty_user_system_content_http_honesty.py b/tests/test_chat_empty_user_system_content_http_honesty.py new file mode 100644 index 000000000..44398dc34 --- /dev/null +++ b/tests/test_chat_empty_user_system_content_http_honesty.py @@ -0,0 +1,154 @@ +"""Chat user/system empty content fail-closed; assistant empty allowed without tools.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_empty_user_system_content_http_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_empty_user_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": " "}], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_content" in blob + assert "non-empty" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_system_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "system", "content": ""}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_nonempty_system_and_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "system", "content": "You are a finance clerk."}, + {"role": "user", "content": "Draft a receipt line."}, + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_assistant_content_in_history() -> None: + """Assistant turns may carry empty content when history only needs the role turn.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": ""}, + {"role": "user", "content": "continue please"}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_unknown_role() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "owner", "content": "hi"}], + }, + ) + assert status == 400, body + assert "invalid_message" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_empty_user_content() + test_http_chat_rejects_empty_system_content() + test_http_chat_accepts_nonempty_system_and_user() + test_http_chat_accepts_empty_assistant_content_in_history() + test_http_chat_rejects_unknown_role() + print("ok") diff --git a/tests/test_chat_include_orchestration_trace_http_honesty.py b/tests/test_chat_include_orchestration_trace_http_honesty.py new file mode 100644 index 000000000..77885daf2 --- /dev/null +++ b/tests/test_chat_include_orchestration_trace_http_honesty.py @@ -0,0 +1,154 @@ +"""Chat Completions include_orchestration_trace honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_include_orchestration_trace_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + # expose_trace_by_default false so omit hides trace unless request opts in. + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, expose_trace_by_default=False), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_include_orchestration_trace_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace string"}], + "include_orchestration_trace": "yes", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_include_orchestration_trace" in blob + assert "boolean" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_include_orchestration_trace_null_as_omit() -> None: + """Explicit JSON null is an SDK optional default — omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace null"}], + "include_orchestration_trace": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_include_orchestration_trace_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace on"}], + "include_orchestration_trace": True, + }, + ) + assert status == 200, body + # Opt-in must surface orchestration for trusted callers. + assert "orchestration" in body or "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_include_orchestration_trace_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace off"}], + "include_orchestration_trace": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_include_orchestration_trace_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no trace flag"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_include_orchestration_trace_non_boolean() + test_http_chat_accepts_include_orchestration_trace_null_as_omit() + test_http_chat_accepts_include_orchestration_trace_true() + test_http_chat_accepts_include_orchestration_trace_false() + test_http_chat_accepts_include_orchestration_trace_omitted() + print("ok") diff --git a/tests/test_chat_include_reject_http_honesty.py b/tests/test_chat_include_reject_http_honesty.py new file mode 100644 index 000000000..71607f13e --- /dev/null +++ b/tests/test_chat_include_reject_http_honesty.py @@ -0,0 +1,84 @@ +"""Chat Completions include field reject honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_include_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_include() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "include"}], + "include": ["usage"], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_include" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_baseline_without_include() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "baseline"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_chat_logit_bias_http_honesty.py b/tests/test_chat_logit_bias_http_honesty.py new file mode 100644 index 000000000..385cd4211 --- /dev/null +++ b/tests/test_chat_logit_bias_http_honesty.py @@ -0,0 +1,172 @@ +"""Chat Completions logit_bias honesty over HTTP (empty no-op; non-empty fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_logit_bias_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_logit_bias() -> None: + """Empty {} is an honest no-op (clients often send empty maps).""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty bias"}], + "logit_bias": {}, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_nonempty_logit_bias() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bias tokens"}], + "logit_bias": {"42": 10.0}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_logit_bias" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_logit_bias_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad type"}], + "logit_bias": [1, 2], + }, + ) + assert status == 400, body + assert "invalid_logit_bias" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_logit_bias() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy empty bias", + "logit_bias": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_nonempty_logit_bias() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy bias", + "logit_bias": {"7": -5}, + }, + ) + assert status == 400, body + assert "invalid_logit_bias" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_logit_bias_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no bias"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_empty_logit_bias() + test_http_chat_rejects_nonempty_logit_bias() + test_http_chat_rejects_logit_bias_array() + test_http_completions_accepts_empty_logit_bias() + test_http_completions_rejects_nonempty_logit_bias() + test_http_chat_accepts_logit_bias_omitted() + print("ok") diff --git a/tests/test_chat_max_completion_tokens_http_honesty.py b/tests/test_chat_max_completion_tokens_http_honesty.py new file mode 100644 index 000000000..5cfc42dcf --- /dev/null +++ b/tests/test_chat_max_completion_tokens_http_honesty.py @@ -0,0 +1,194 @@ +"""Chat Completions max_completion_tokens honesty over HTTP (budget precedence).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_max_completion_tokens_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_max_completion_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "budget 64"}], + "max_completion_tokens": 64, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_max_completion_tokens_zero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "zero budget"}], + "max_completion_tokens": 0, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_max_completion_tokens" in blob + assert "positive" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_max_completion_tokens_bool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bool budget"}], + "max_completion_tokens": True, + }, + ) + assert status == 400, body + assert "invalid_max_completion_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_max_completion_tokens_too_large() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "huge budget"}], + "max_completion_tokens": 1_048_577, + }, + ) + assert status == 400, body + assert "invalid_max_completion_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_prefers_max_completion_tokens_when_both_set() -> None: + """When both budgets are present, request must still succeed (max_completion wins).""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "both budgets"}], + "max_tokens": 8, + "max_completion_tokens": 32, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_invalid_max_tokens_when_only_legacy() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "legacy zero"}], + "max_tokens": 0, + }, + ) + assert status == 400, body + assert "invalid_max_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_max_completion_tokens_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no budget field"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_max_completion_tokens() + test_http_chat_rejects_max_completion_tokens_zero() + test_http_chat_rejects_max_completion_tokens_bool() + test_http_chat_rejects_max_completion_tokens_too_large() + test_http_chat_prefers_max_completion_tokens_when_both_set() + test_http_chat_rejects_invalid_max_tokens_when_only_legacy() + test_http_chat_accepts_max_completion_tokens_omitted() + print("ok") diff --git a/tests/test_chat_max_tool_calls_http_honesty.py b/tests/test_chat_max_tool_calls_http_honesty.py new file mode 100644 index 000000000..48919b0a8 --- /dev/null +++ b/tests/test_chat_max_tool_calls_http_honesty.py @@ -0,0 +1,136 @@ +"""Chat Completions max_tool_calls honesty: null/empty omit; else fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_max_tool_calls_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_max_tool_calls_null_and_empty_string() -> None: + server, thread, port = _server() + try: + for value in (None, "", " "): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello"}], + "max_tool_calls": value, + }, + ) + assert status == 200, (value, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_max_tool_calls_nonzero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello"}], + "max_tool_calls": 3, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_max_tool_calls" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_max_tool_calls_one() -> None: + """Even max_tool_calls=1 is unsupported — no tool loop on chat passthrough.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello"}], + "max_tool_calls": 1, + }, + ) + assert status == 400, body + assert "invalid_max_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_max_tool_calls_false_zero() -> None: + server, thread, port = _server() + try: + for value in (False, 0, True): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello"}], + "max_tool_calls": value, + }, + ) + assert status == 400, (value, body) + assert "invalid_max_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_max_tool_calls_null_and_empty_string() + test_http_chat_rejects_max_tool_calls_nonzero() + test_http_chat_rejects_max_tool_calls_one() + test_http_chat_rejects_max_tool_calls_false_zero() + print("ok") diff --git a/tests/test_chat_message_name_http_honesty.py b/tests/test_chat_message_name_http_honesty.py new file mode 100644 index 000000000..7c5e34413 --- /dev/null +++ b/tests/test_chat_message_name_http_honesty.py @@ -0,0 +1,198 @@ +"""Chat message participant name honesty over HTTP (OpenAI optional name field).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_message_name_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_user_message_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hello", "name": "buyer_alpha"}, + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_system_and_assistant_names() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "system", "content": "Be concise.", "name": "policy_bot"}, + {"role": "user", "content": "hi", "name": "user_1"}, + {"role": "assistant", "content": "hello", "name": "assistant_a"}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_message_name_as_omit() -> None: + """Empty/whitespace participant name is SDK omit-equivalent.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi", "name": " "}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_message_name_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi", "name": "a" * 65}], + }, + ) + assert status == 400, body + assert "invalid_message_name" in json.dumps(body) + assert "64" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_message_name_bad_charset() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi", "name": "bad name!"}], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_name" in blob + assert "a-zA-Z0-9" in blob or "match" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_name_on_tool_message() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + { + "role": "tool", + "content": "result", + "tool_call_id": "call_1", + "name": "should_not_be_here", + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_name" in blob + assert "tool" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_name_with_underscore_hyphen() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "ping", "name": "Buyer-Agent_01"}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_user_message_name() + test_http_chat_accepts_system_and_assistant_names() + test_http_chat_accepts_empty_message_name_as_omit() + test_http_chat_rejects_message_name_too_long() + test_http_chat_rejects_message_name_bad_charset() + test_http_chat_rejects_name_on_tool_message() + test_http_chat_accepts_name_with_underscore_hyphen() + print("ok") diff --git a/tests/test_chat_modalities_http_honesty.py b/tests/test_chat_modalities_http_honesty.py new file mode 100644 index 000000000..ee7f19ffc --- /dev/null +++ b/tests/test_chat_modalities_http_honesty.py @@ -0,0 +1,167 @@ +"""Chat Completions modalities honesty over HTTP (text-only gateway).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_modalities_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_modalities_text_only() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "text only"}], + "modalities": ["text"], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_modalities_audio() -> None: + """Buyers must not believe audio output was produced by a text-only gateway.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "speak"}], + "modalities": ["audio"], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_modalities" in blob + assert "text" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_modalities_text_and_audio() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "both"}], + "modalities": ["text", "audio"], + }, + ) + assert status == 400, body + assert "invalid_modalities" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_modalities_as_omit() -> None: + """Empty modalities [] is an omit-equivalent SDK optional default.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty mods"}], + "modalities": [], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_modalities_non_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "string mods"}], + "modalities": "text", + }, + ) + assert status == 400, body + assert "invalid_modalities" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_modalities_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "default modalities"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_modalities_text_only() + test_http_chat_rejects_modalities_audio() + test_http_chat_rejects_modalities_text_and_audio() + test_http_chat_accepts_empty_modalities_as_omit() + test_http_chat_rejects_modalities_non_array() + test_http_chat_accepts_modalities_omitted() + print("ok") diff --git a/tests/test_chat_n_gt1_http_honesty.py b/tests/test_chat_n_gt1_http_honesty.py new file mode 100644 index 000000000..19bcb8ed4 --- /dev/null +++ b/tests/test_chat_n_gt1_http_honesty.py @@ -0,0 +1,176 @@ +"""Chat/Completions n>1 honesty over HTTP (single choice only).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_n_gt1_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_n_gt1() -> None: + """Buyers must not receive a single choice when they paid for n=3.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "give three"}], + "n": 3, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_n" in blob + assert "not supported" in blob + assert "chat" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_n_one() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "n is one"}], + "n": 1, + }, + ) + assert status == 200, body + assert len(body.get("choices") or []) == 1 + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_n_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no n"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_n_zero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "n zero"}], + "n": 0, + }, + ) + assert status == 400, body + assert "invalid_n" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_n_gt1() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy n", + "n": 2, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_n" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_n_bool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "n bool"}], + "n": True, + }, + ) + assert status == 400, body + assert "invalid_n" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_n_gt1() + test_http_chat_accepts_n_one() + test_http_chat_accepts_n_omitted() + test_http_chat_rejects_n_zero() + test_http_completions_rejects_n_gt1() + test_http_chat_rejects_n_bool() + print("ok") diff --git a/tests/test_chat_openai_metadata_http_honesty.py b/tests/test_chat_openai_metadata_http_honesty.py new file mode 100644 index 000000000..322d1072a --- /dev/null +++ b/tests/test_chat_openai_metadata_http_honesty.py @@ -0,0 +1,216 @@ +"""Chat/Completions OpenAI metadata shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_openai_metadata_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_string_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta ok"}], + "metadata": {"request_id": "req-1", "tenant": "acme"}, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta string"}], + "metadata": "not-an-object", + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_non_string_value() -> None: + """Buyers must not store untyped junk that observability would silently drop.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta int value"}], + "metadata": {"count": 3}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_metadata" in blob + assert "strings" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_too_many_entries() -> None: + server, thread, port = _server() + try: + meta = {f"k{i:02d}": f"v{i}" for i in range(17)} + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta overflow"}], + "metadata": meta, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + assert "16" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_key_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta key long"}], + "metadata": {"k" * 65: "v"}, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + assert "64" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_metadata_value_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "meta value long"}], + "metadata": {"k": "v" * 513}, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + assert "512" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_string_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy meta", + "metadata": {"source": "cli"}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_metadata_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no meta"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_string_metadata() + test_http_chat_rejects_metadata_non_object() + test_http_chat_rejects_metadata_non_string_value() + test_http_chat_rejects_metadata_too_many_entries() + test_http_chat_rejects_metadata_key_too_long() + test_http_chat_rejects_metadata_value_too_long() + test_http_completions_accepts_string_metadata() + test_http_chat_accepts_metadata_omitted() + print("ok") diff --git a/tests/test_chat_orchestration_mode_http_honesty.py b/tests/test_chat_orchestration_mode_http_honesty.py new file mode 100644 index 000000000..8ce21f8e6 --- /dev/null +++ b/tests/test_chat_orchestration_mode_http_honesty.py @@ -0,0 +1,136 @@ +"""Chat Completions mode/orchestration_mode: auto|route|conduct; invalid fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_orchestration_mode_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ + ModelAgent("planner_agent", "mock-planner", tags=("planning", "reasoning")), + ModelAgent("builder_agent", "mock-builder", tags=("coding", "writing")), + ModelAgent("reviewer_agent", "mock-reviewer", tags=("verification", "review")), + ] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def test_http_chat_accepts_mode_route() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "mode": "route", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_orchestration_mode_auto() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "orchestration_mode": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_invalid_mode() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "mode": "cascade", + }, + ) + assert status == 400, body + assert "invalid_mode" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_mode_non_string() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "say hi"}], + "orchestration": 1, + }, + ) + assert status == 400, body + assert "invalid_mode" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_mode_route() + test_http_chat_accepts_orchestration_mode_auto() + test_http_chat_rejects_invalid_mode() + test_http_chat_rejects_mode_non_string() + print("ok") diff --git a/tests/test_chat_parallel_tool_calls_http_honesty.py b/tests/test_chat_parallel_tool_calls_http_honesty.py new file mode 100644 index 000000000..8ad66c481 --- /dev/null +++ b/tests/test_chat_parallel_tool_calls_http_honesty.py @@ -0,0 +1,146 @@ +"""Chat parallel_tool_calls honesty: boolean type; true requires tools.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_parallel_tool_calls_honesty_token" # noqa: S105 + +_TOOLS = [ + { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up an invoice by id", + "parameters": { + "type": "object", + "properties": {"invoice_id": {"type": "string"}}, + "required": ["invoice_id"], + }, + }, + } +] + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_parallel_tool_calls_false_without_tools_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "parallel_tool_calls": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_parallel_tool_calls_true_without_tools_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "parallel_tool_calls": True, + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_parallel_tool_calls_non_boolean_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "parallel_tool_calls": "yes", + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_parallel_tool_calls_true_with_tools_passthrough() -> None: + """With tools, parallel_tool_calls triggers single-agent passthrough path.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "lookup invoice 42"}], + "tools": _TOOLS, + "parallel_tool_calls": True, + }, + ) + # Mock passthrough returns chat-shaped body + assert status == 200, body + assert "choices" in body or "id" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_parallel_tool_calls_false_without_tools_ok() + test_http_chat_parallel_tool_calls_true_without_tools_fail_closed() + test_http_chat_parallel_tool_calls_non_boolean_fail_closed() + test_http_chat_parallel_tool_calls_true_with_tools_passthrough() + print("ok") diff --git a/tests/test_chat_penalties_http_honesty.py b/tests/test_chat_penalties_http_honesty.py new file mode 100644 index 000000000..0a6b5d193 --- /dev/null +++ b/tests/test_chat_penalties_http_honesty.py @@ -0,0 +1,171 @@ +"""Chat/Completions presence_penalty and frequency_penalty honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_penalties_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_presence_and_frequency_penalty() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "penalties in range"}], + "presence_penalty": 0.5, + "frequency_penalty": -0.25, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_presence_penalty_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "presence high"}], + "presence_penalty": 2.5, + }, + ) + assert status == 400, body + assert "invalid_presence_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_frequency_penalty_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "frequency low"}], + "frequency_penalty": -3, + }, + ) + assert status == 400, body + assert "invalid_frequency_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_presence_penalty_non_number() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "presence bool"}], + "presence_penalty": True, + }, + ) + assert status == 400, body + assert "invalid_presence_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_penalties_in_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy penalties", + "presence_penalty": 1.0, + "frequency_penalty": 1.0, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_penalties_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no penalties"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_presence_and_frequency_penalty() + test_http_chat_rejects_presence_penalty_out_of_range() + test_http_chat_rejects_frequency_penalty_out_of_range() + test_http_chat_rejects_presence_penalty_non_number() + test_http_completions_accepts_penalties_in_range() + test_http_chat_accepts_penalties_omitted() + print("ok") diff --git a/tests/test_chat_prediction_http_honesty.py b/tests/test_chat_prediction_http_honesty.py new file mode 100644 index 000000000..e50fc800d --- /dev/null +++ b/tests/test_chat_prediction_http_honesty.py @@ -0,0 +1,152 @@ +"""Chat Completions prediction (Predicted Outputs) honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_prediction_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_prediction_object() -> None: + """Buyers must not believe Predicted Outputs latency optimization was applied.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict me"}], + "prediction": { + "type": "content", + "content": "expected completion text", + }, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_prediction" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_prediction_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict string"}], + "prediction": "expected", + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_prediction_null_as_omit() -> None: + """Explicit JSON null is an SDK optional default — omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict null"}], + "prediction": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_prediction_bool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict bool"}], + "prediction": True, + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_prediction_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no prediction"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_prediction_object() + test_http_chat_rejects_prediction_string() + test_http_chat_accepts_prediction_null_as_omit() + test_http_chat_rejects_prediction_bool() + test_http_chat_accepts_prediction_omitted() + print("ok") diff --git a/tests/test_chat_reasoning_effort_http_honesty.py b/tests/test_chat_reasoning_effort_http_honesty.py new file mode 100644 index 000000000..a6d48ed3a --- /dev/null +++ b/tests/test_chat_reasoning_effort_http_honesty.py @@ -0,0 +1,150 @@ +"""Chat Completions reasoning_effort honesty over HTTP (not applied on route path).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_reasoning_effort_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_reasoning_effort_high() -> None: + """Buyers must not believe high reasoning was applied when the knob is dropped.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "think hard"}], + "reasoning_effort": "high", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_reasoning_effort" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_reasoning_effort_low() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "think low"}], + "reasoning_effort": "low", + }, + ) + assert status == 400, body + assert "invalid_reasoning_effort" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_reasoning_effort_none_as_omit() -> None: + """OpenAI none disables extra reasoning — honest omit no-op on this gateway.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "think none"}], + "reasoning_effort": "none", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_reasoning_effort_bool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "think bool"}], + "reasoning_effort": True, + }, + ) + assert status == 400, body + assert "invalid_reasoning_effort" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_reasoning_effort_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no reasoning knob"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_reasoning_effort_high() + test_http_chat_rejects_reasoning_effort_low() + test_http_chat_accepts_reasoning_effort_none_as_omit() + test_http_chat_rejects_reasoning_effort_bool() + test_http_chat_accepts_reasoning_effort_omitted() + print("ok") diff --git a/tests/test_chat_reasoning_object_reject_http_honesty.py b/tests/test_chat_reasoning_object_reject_http_honesty.py new file mode 100644 index 000000000..716128be7 --- /dev/null +++ b/tests/test_chat_reasoning_object_reject_http_honesty.py @@ -0,0 +1,103 @@ +"""Chat Completions reasoning object reject honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_reasoning_object_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_without_reasoning_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no reasoning object"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_reasoning_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "with reasoning object"}], + "reasoning": {"effort": "high"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_reasoning" in blob + assert "unknown_fields" not in blob + assert "responses" in blob.lower() or "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_reasoning_empty_object_as_omit() -> None: + """Empty reasoning {} is an omit-equivalent SDK optional default.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty reasoning"}], + "reasoning": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_chat_response_format_http_honesty.py b/tests/test_chat_response_format_http_honesty.py new file mode 100644 index 000000000..59b194a60 --- /dev/null +++ b/tests/test_chat_response_format_http_honesty.py @@ -0,0 +1,218 @@ +"""Chat Completions response_format honesty over HTTP (structured-output shape).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_response_format_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_response_format_text() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "plain text"}], + "response_format": {"type": "text"}, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_response_format_json_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "json object mode"}], + "response_format": {"type": "json_object"}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_valid_json_schema_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "structured"}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "receipt_line", + "schema": { + "type": "object", + "properties": {"amount": {"type": "number"}}, + }, + "strict": True, + }, + }, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_unknown_response_format_type() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad type"}], + "response_format": {"type": "xml"}, + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_json_object_with_sibling_keys() -> None: + """Buyers must not smuggle extra fields into type-only response_format objects.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "sibling"}], + "response_format": {"type": "json_object", "strict": True}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "only the type field" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_json_schema_without_schema_body() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "missing schema"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "receipt_line"}, + }, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "schema must be an object" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_non_object_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "string fmt"}], + "response_format": "json", + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_response_format_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no format"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_response_format_text() + test_http_chat_accepts_response_format_json_object() + test_http_chat_accepts_valid_json_schema_response_format() + test_http_chat_rejects_unknown_response_format_type() + test_http_chat_rejects_json_object_with_sibling_keys() + test_http_chat_rejects_json_schema_without_schema_body() + test_http_chat_rejects_non_object_response_format() + test_http_chat_accepts_response_format_omitted() + print("ok") diff --git a/tests/test_chat_service_tier_http_honesty.py b/tests/test_chat_service_tier_http_honesty.py new file mode 100644 index 000000000..7b7477341 --- /dev/null +++ b/tests/test_chat_service_tier_http_honesty.py @@ -0,0 +1,192 @@ +"""Chat Completions service_tier honesty over HTTP (capacity priority knob).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_service_tier_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_service_tier_auto() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "auto tier"}], + "service_tier": "auto", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_service_tier_default() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "default tier"}], + "service_tier": "default", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_service_tier_flex() -> None: + """flex/priority are capacity modes this gateway does not apply — fail closed.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "flex tier"}], + "service_tier": "flex", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_service_tier" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_service_tier_priority() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "priority tier"}], + "service_tier": "priority", + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_service_tier_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad tier type"}], + "service_tier": 1, + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_service_tier_flex() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy flex", + "service_tier": "flex", + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_service_tier_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no tier"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_service_tier_auto() + test_http_chat_accepts_service_tier_default() + test_http_chat_rejects_service_tier_flex() + test_http_chat_rejects_service_tier_priority() + test_http_chat_rejects_service_tier_non_string() + test_http_completions_rejects_service_tier_flex() + test_http_chat_accepts_service_tier_omitted() + print("ok") diff --git a/tests/test_chat_store_http_honesty.py b/tests/test_chat_store_http_honesty.py new file mode 100644 index 000000000..87b8ccb1d --- /dev/null +++ b/tests/test_chat_store_http_honesty.py @@ -0,0 +1,149 @@ +"""Chat Completions store honesty over HTTP (OpenAI store persistence knob).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_store_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_store_true() -> None: + """Buyers must not believe store=true persisted a completion when it cannot.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "remember this"}], + "store": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_store" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_store_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no store please"}], + "store": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_store_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "default store omit"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_store_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad store type"}], + "store": "yes", + }, + ) + assert status == 400, body + assert "invalid_store" in json.dumps(body) + assert "boolean" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_store_null_as_omit() -> None: + """Explicit JSON null is an SDK optional default — omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null store"}], + "store": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_store_true() + test_http_chat_accepts_store_false() + test_http_chat_accepts_store_omitted() + test_http_chat_rejects_store_non_boolean() + test_http_chat_accepts_store_null_as_omit() + print("ok") diff --git a/tests/test_chat_stream_options_http_honesty.py b/tests/test_chat_stream_options_http_honesty.py new file mode 100644 index 000000000..7f09402a1 --- /dev/null +++ b/tests/test_chat_stream_options_http_honesty.py @@ -0,0 +1,177 @@ +"""Chat stream_options honesty: requires stream=true; include_usage true fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_stream_options_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict | str]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + try: + return response.status, json.loads(raw) + except json.JSONDecodeError: + return response.status, raw + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8") + try: + return exc.code, json.loads(raw) + except json.JSONDecodeError: + return exc.code, raw + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_stream_options_all_false_without_stream_as_omit() -> None: + """All-false stream_options without stream is an omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream_options": {"include_usage": False}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_stream_options_true_without_stream_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_stream_options_include_usage_true_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_stream_options_include_usage_false_with_stream_ok() -> None: + """stream=true with include_usage=false is accepted (usage not requested).""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": False}, + }, + ) + # Streaming may return 200 SSE body; accept 200 + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_stream_options_non_object_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": "include_usage", + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_omits_stream_options_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-generalist", + "messages": [{"role": "user", "content": "hi"}], + }, + ) + assert status == 200, body + assert isinstance(body, dict) and "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_stream_options_all_false_without_stream_as_omit() + test_http_chat_stream_options_true_without_stream_fail_closed() + test_http_chat_stream_options_include_usage_true_fail_closed() + test_http_chat_stream_options_include_usage_false_with_stream_ok() + test_http_chat_stream_options_non_object_fail_closed() + test_http_chat_omits_stream_options_ok() + print("ok") diff --git a/tests/test_chat_temperature_top_p_http_honesty.py b/tests/test_chat_temperature_top_p_http_honesty.py new file mode 100644 index 000000000..c080a7ed3 --- /dev/null +++ b/tests/test_chat_temperature_top_p_http_honesty.py @@ -0,0 +1,212 @@ +"""Chat/Completions temperature and top_p sampling honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_temperature_top_p_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_temperature_and_top_p_in_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "sample in range"}], + "temperature": 0.7, + "top_p": 0.9, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_temperature_above_two() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hot"}], + "temperature": 2.5, + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_temperature_negative() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "cold"}], + "temperature": -0.1, + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_temperature_bool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bool temp"}], + "temperature": True, + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_top_p_zero() -> None: + """top_p must be in (0, 1]; zero is not a valid nucleus mass.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "top_p zero"}], + "top_p": 0, + }, + ) + assert status == 400, body + assert "invalid_top_p" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_top_p_above_one() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "top_p high"}], + "top_p": 1.1, + }, + ) + assert status == 400, body + assert "invalid_top_p" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_temperature_bounds() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy temp bounds", + "temperature": 0, + "top_p": 1, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_sampling_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "defaults"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_temperature_and_top_p_in_range() + test_http_chat_rejects_temperature_above_two() + test_http_chat_rejects_temperature_negative() + test_http_chat_rejects_temperature_bool() + test_http_chat_rejects_top_p_zero() + test_http_chat_rejects_top_p_above_one() + test_http_completions_accepts_temperature_bounds() + test_http_chat_accepts_sampling_omitted() + print("ok") diff --git a/tests/test_chat_tool_call_id_http_honesty.py b/tests/test_chat_tool_call_id_http_honesty.py new file mode 100644 index 000000000..ab0978338 --- /dev/null +++ b/tests/test_chat_tool_call_id_http_honesty.py @@ -0,0 +1,170 @@ +"""Chat Completions tool message tool_call_id honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_tool_call_id_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_tool_message_with_tool_call_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + { + "role": "tool", + "content": "result payload", + "tool_call_id": "call_abc123", + }, + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_message_missing_tool_call_id() -> None: + """Buyers must not bind tool results without a tool_call_id.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + {"role": "tool", "content": "orphan result"}, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob + assert "tool_call_id" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_message_blank_tool_call_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + {"role": "tool", "content": "blank id", "tool_call_id": " "}, + ], + }, + ) + assert status == 400, body + assert "tool_call_id" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_call_id_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + { + "role": "tool", + "content": "long id", + "tool_call_id": "c" * 129, + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "tool_call_id" in blob + assert "128" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_call_id_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "run tool"}, + {"role": "tool", "content": "num id", "tool_call_id": 42}, + ], + }, + ) + assert status == 400, body + assert "tool_call_id" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_tool_message_with_tool_call_id() + test_http_chat_rejects_tool_message_missing_tool_call_id() + test_http_chat_rejects_tool_message_blank_tool_call_id() + test_http_chat_rejects_tool_call_id_too_long() + test_http_chat_rejects_tool_call_id_non_string() + print("ok") diff --git a/tests/test_chat_tool_choice_functions_http_honesty.py b/tests/test_chat_tool_choice_functions_http_honesty.py new file mode 100644 index 000000000..372b9acef --- /dev/null +++ b/tests/test_chat_tool_choice_functions_http_honesty.py @@ -0,0 +1,171 @@ +"""Chat tools honesty: functions/function_call rejected; tool_choice required/named requires tools; auto/none without tools are no-ops.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_tool_choice_functions_http_honesty_token" # noqa: S105 + +_TOOLS = [ + { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up invoice by id", + "parameters": { + "type": "object", + "properties": {"invoice_id": {"type": "string"}}, + "required": ["invoice_id"], + }, + }, + } +] + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_functions_legacy_surface() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lookup invoice 9"}], + "functions": [ + { + "name": "lookup_invoice", + "description": "legacy", + "parameters": {"type": "object", "properties": {}}, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_functions" in blob + assert "tools" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_function_call_auto_without_functions_as_omit() -> None: + """function_call none/auto without functions is an omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lookup invoice 9"}], + "function_call": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_function_call_named_without_tools_migration() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lookup invoice 9"}], + "function_call": {"name": "lookup"}, + }, + ) + assert status == 400, body + assert "invalid_functions" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_tool_choice_auto_without_tools_as_omit() -> None: + """tool_choice auto/none without tools is an omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi"}], + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_tools_with_tool_choice_passthrough_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lookup invoice 9"}], + "tools": _TOOLS, + "tool_choice": "auto", + }, + ) + assert status == 200, body + assert "choices" in body or "id" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_functions_legacy_surface() + test_http_chat_accepts_function_call_auto_without_functions_as_omit() + test_http_chat_rejects_function_call_named_without_tools_migration() + test_http_chat_accepts_tool_choice_auto_without_tools_as_omit() + test_http_chat_tools_with_tool_choice_passthrough_ok() + print("ok") diff --git a/tests/test_chat_tools_shape_http_honesty.py b/tests/test_chat_tools_shape_http_honesty.py new file mode 100644 index 000000000..fa4952ad4 --- /dev/null +++ b/tests/test_chat_tools_shape_http_honesty.py @@ -0,0 +1,245 @@ +"""Chat Completions tools array shape honesty over HTTP (before passthrough).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_tools_shape_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _base_messages(): + return [{"role": "user", "content": "use a tool"}] + + +def test_http_chat_accepts_valid_function_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": _base_messages(), + "tools": [ + { + "type": "function", + "function": { + "name": "lookup_balance", + "description": "Fetch account balance", + "parameters": { + "type": "object", + "properties": {"account_id": {"type": "string"}}, + }, + }, + } + ], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_tools_array_as_noop() -> None: + """SDKs often send tools: [] when no tools are configured — honest no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": _base_messages(), + "tools": [], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_type_not_function() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": _base_messages(), + "tools": [{"type": "code_interpreter", "function": {"name": "x"}}], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_missing_function_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": _base_messages(), + "tools": [{"type": "function", "function": {"description": "no name"}}], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + assert "name" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_function_name_bad_charset() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": _base_messages(), + "tools": [ + { + "type": "function", + "function": {"name": "bad name!"}, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "a-zA-Z0-9" in blob or "match" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_sibling_unknown_fields() -> None: + """Extra siblings on tool objects must fail closed, not silently pass through.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": _base_messages(), + "tools": [ + { + "type": "function", + "function": {"name": "lookup_balance"}, + "extra_flag": True, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "only type and function" in blob or "extra_flag" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_parameters_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": _base_messages(), + "tools": [ + { + "type": "function", + "function": { + "name": "lookup_balance", + "parameters": "not-a-schema", + }, + } + ], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + assert "parameters" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_tools_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no tools"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_valid_function_tools() + test_http_chat_rejects_empty_tools_array() + test_http_chat_rejects_tool_type_not_function() + test_http_chat_rejects_tool_missing_function_name() + test_http_chat_rejects_tool_function_name_bad_charset() + test_http_chat_rejects_tool_sibling_unknown_fields() + test_http_chat_rejects_parameters_non_object() + test_http_chat_accepts_tools_omitted() + print("ok") diff --git a/tests/test_chat_top_logprobs_http_honesty.py b/tests/test_chat_top_logprobs_http_honesty.py new file mode 100644 index 000000000..feb157e30 --- /dev/null +++ b/tests/test_chat_top_logprobs_http_honesty.py @@ -0,0 +1,171 @@ +"""Chat Completions logprobs/top_logprobs honesty over HTTP (not returned by route).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_top_logprobs_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_logprobs_true() -> None: + """Buyers must not believe token logprobs were returned when route never emits them.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "logprobs true"}], + "logprobs": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_logprobs" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_logprobs_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "logprobs false"}], + "logprobs": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_top_logprobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "top logprobs"}], + "top_logprobs": 5, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_top_logprobs" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_top_logprobs_with_logprobs_false() -> None: + """Even with logprobs=false, top_logprobs must not silently no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "both knobs"}], + "logprobs": False, + "top_logprobs": 2, + }, + ) + assert status == 400, body + assert "invalid_top_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_logprobs_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "logprobs int"}], + "logprobs": 5, + }, + ) + assert status == 400, body + assert "invalid_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_logprobs_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no logprobs"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_logprobs_true() + test_http_chat_accepts_logprobs_false() + test_http_chat_rejects_top_logprobs() + test_http_chat_rejects_top_logprobs_with_logprobs_false() + test_http_chat_rejects_logprobs_non_boolean() + test_http_chat_accepts_logprobs_omitted() + print("ok") diff --git a/tests/test_chat_unknown_fields_http_honesty.py b/tests/test_chat_unknown_fields_http_honesty.py new file mode 100644 index 000000000..ef870d068 --- /dev/null +++ b/tests/test_chat_unknown_fields_http_honesty.py @@ -0,0 +1,205 @@ +"""Chat/Completions unknown request fields honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_unknown_fields_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_unknown_request_field() -> None: + """Buyers must not believe unsupported OpenAI-adjacent knobs were applied.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "unknown knob"}], + "unsupported_client_knob": "demo-1", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "unknown_fields" in blob + assert "unsupported_client_knob" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_multiple_unknown_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "two unknowns"}], + "unsupported_client_knob": "cache-1", + "another_unsupported_knob": "safety-1", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "unknown_fields" in blob + assert "unsupported_client_knob" in blob or "safety_identifier" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_unknown_request_field() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy unknown", + "not_a_real_completions_field": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "unknown_fields" in blob + assert "not_a_real_completions_field" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_audio_with_named_error() -> None: + """audio is allowed as a named unsupported field (not opaque unknown_fields).""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy audio", + "audio": {"voice": "alloy"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_audio" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_stream_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stream string"}], + "stream": "yes", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "stream must be a boolean" in blob or "invalid_request" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_invalid_mode() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad mode"}], + "mode": "turbo", + }, + ) + assert status == 400, body + assert "invalid_mode" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_known_fields_only() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "known only"}], + "temperature": 0.5, + "mode": "route", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_unknown_request_field() + test_http_chat_rejects_multiple_unknown_fields() + test_http_completions_rejects_unknown_request_field() + test_http_completions_rejects_audio_with_named_error() + test_http_chat_rejects_stream_non_boolean() + test_http_chat_rejects_invalid_mode() + test_http_chat_accepts_known_fields_only() + print("ok") diff --git a/tests/test_commercial_readiness.py b/tests/test_commercial_readiness.py index a07f3ebe7..b357fde9f 100644 --- a/tests/test_commercial_readiness.py +++ b/tests/test_commercial_readiness.py @@ -172,7 +172,7 @@ def test_commercial_readiness_endpoint_openapi_admin_and_docs_contract() -> None ) chat_status, _ = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "Analyze, verify, and summarize commercial readiness."}]}, + {"model": "mock-planner", "messages": [{"role": "user", "content": "Analyze, verify, and summarize commercial readiness."}]}, "inference_secret", ) readiness_status, readiness = get_json( diff --git a/tests/test_completions_chat_era_fields_reject_http_honesty.py b/tests/test_completions_chat_era_fields_reject_http_honesty.py new file mode 100644 index 000000000..6a7693dc0 --- /dev/null +++ b/tests/test_completions_chat_era_fields_reject_http_honesty.py @@ -0,0 +1,140 @@ +"""Completions chat-era modalities/prediction/reasoning_effort reject honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_chat_era_fields_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_baseline_without_chat_era_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "prompt": "hello no chat-era fields"} + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_modalities_text_as_noop() -> None: + """Text-only modalities is an honest no-op on this text Completions path.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello modalities text", + "modalities": ["text"], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_non_text_modalities() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello modalities audio", + "modalities": ["audio"], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_chat_era_field" in blob + assert "chat/completions" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_prediction() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello prediction", + "prediction": {"type": "content", "content": "partial"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_chat_era_field" in blob + assert "chat/completions" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_reasoning_effort() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello reasoning", + "reasoning_effort": "high", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_chat_era_field" in blob + assert "chat/completions" in blob + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_empty_tools_noop_http_honesty.py b/tests/test_completions_empty_tools_noop_http_honesty.py new file mode 100644 index 000000000..597dceb0c --- /dev/null +++ b/tests/test_completions_empty_tools_noop_http_honesty.py @@ -0,0 +1,107 @@ +"""Completions empty tools array no-op honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_empty_tools_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_empty_tools_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "empty tools", "tools": []}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_nonempty_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "nonempty tools", + "tools": [ + { + "type": "function", + "function": {"name": "lookup_item", "parameters": {}}, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "chat/completions" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_tool_choice_none_with_empty_tools() -> None: + """none/auto/empty tool_choice with tools=[] are omit-equivalent SDK defaults.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "tool choice none empty tools", + "tools": [], + "tool_choice": "none", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_include_reject_http_honesty.py b/tests/test_completions_include_reject_http_honesty.py new file mode 100644 index 000000000..038ce3600 --- /dev/null +++ b/tests/test_completions_include_reject_http_honesty.py @@ -0,0 +1,80 @@ +"""Completions include field reject honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_include_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_rejects_include() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "include field", + "include": ["logprobs"], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_include" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_baseline_without_include() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "prompt": "baseline"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_legacy_knobs_http_honesty.py b/tests/test_completions_legacy_knobs_http_honesty.py new file mode 100644 index 000000000..6281bca7c --- /dev/null +++ b/tests/test_completions_legacy_knobs_http_honesty.py @@ -0,0 +1,204 @@ +"""Completions best_of/echo/suffix/logprobs honesty: HTTP fail-closed contracts.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_legacy_knobs_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_baseline_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "draft a one-line payment receipt"}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_best_of_one_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "best_of": 1}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_best_of_multi_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "best_of": 3}, + ) + assert status == 400, body + assert "invalid_best_of" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_echo_false_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "echo": False}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_echo_true_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "echo": True}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_echo" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_empty_suffix_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "suffix": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_nonempty_suffix_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "function head(", + "suffix": ") { return 1; }", + }, + ) + assert status == 400, body + assert "invalid_suffix" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_logprobs_false_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "logprobs": False}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_logprobs_integer_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "logprobs": 5}, + ) + assert status == 400, body + assert "invalid_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_logprobs_true_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "logprobs": True}, + ) + assert status == 400, body + assert "invalid_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_baseline_ok() + test_http_completions_best_of_one_ok() + test_http_completions_best_of_multi_fail_closed() + test_http_completions_echo_false_ok() + test_http_completions_echo_true_fail_closed() + test_http_completions_empty_suffix_ok() + test_http_completions_nonempty_suffix_fail_closed() + test_http_completions_logprobs_false_ok() + test_http_completions_logprobs_integer_fail_closed() + test_http_completions_logprobs_true_fail_closed() + print("ok") diff --git a/tests/test_completions_max_completion_tokens_http_honesty.py b/tests/test_completions_max_completion_tokens_http_honesty.py new file mode 100644 index 000000000..533c78784 --- /dev/null +++ b/tests/test_completions_max_completion_tokens_http_honesty.py @@ -0,0 +1,147 @@ +"""Completions max_completion_tokens honesty over HTTP (chat-era alias).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_max_completion_tokens_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_max_completion_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello mct", + "max_completion_tokens": 64, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_prefers_max_completion_tokens_over_max_tokens() -> None: + server, thread, port = _server() + try: + # Both present: max_completion_tokens wins (chat-era precedence). + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello both", + "max_tokens": 8, + "max_completion_tokens": 128, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_zero_max_completion_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello zero", + "max_completion_tokens": 0, + }, + ) + assert status == 400, body + assert "invalid_max_completion_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_non_integer_max_completion_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello float", + "max_completion_tokens": 1.5, + }, + ) + assert status == 400, body + assert "invalid_max_completion_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_accepts_legacy_max_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello legacy", + "max_tokens": 32, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_max_completion_tokens() + test_http_completions_prefers_max_completion_tokens_over_max_tokens() + test_http_completions_rejects_zero_max_completion_tokens() + test_http_completions_rejects_non_integer_max_completion_tokens() + test_http_completions_still_accepts_legacy_max_tokens() + print("ok") diff --git a/tests/test_completions_max_tokens_http_honesty.py b/tests/test_completions_max_tokens_http_honesty.py new file mode 100644 index 000000000..4f6e6870e --- /dev/null +++ b/tests/test_completions_max_tokens_http_honesty.py @@ -0,0 +1,135 @@ +"""Completions max_tokens is applied to the provider client for the request.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + SecurityConfig, + build_server, +) + +_TEST_AUTH_TOKEN = "cmpl_max_tokens_pass_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def test_http_max_tokens_applies_and_restores() -> None: + orch = build() + default_cap = orch.client.max_output_tokens + server = build_server(orch, port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": 64}, + ) + assert status == 200, body + assert body["object"] == "text_completion" + # Restored after request so later work uses the server default again. + assert orch.client.max_output_tokens == default_cap + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_rejects_non_positive_max_tokens() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": 0}, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_max_tokens" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_without_max_tokens_ok() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post(port, {"model": "mock-planner", "prompt": "hello"}) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_max_tokens_applies_and_restores() + test_http_rejects_non_positive_max_tokens() + test_http_without_max_tokens_ok() + + +def test_http_rejects_bool_max_tokens() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": True}, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_max_tokens" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_rejects_oversized_max_tokens() -> None: + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello", "max_tokens": 2_000_000}, + ) + assert status == 400, body + assert body["error"]["code"] == "invalid_max_tokens" + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_max_tool_calls_http_honesty.py b/tests/test_completions_max_tool_calls_http_honesty.py new file mode 100644 index 000000000..d97cb5ffc --- /dev/null +++ b/tests/test_completions_max_tool_calls_http_honesty.py @@ -0,0 +1,136 @@ +"""Legacy Completions max_tool_calls honesty: null/empty omit; else fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_max_tool_calls_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_max_tool_calls_null_and_empty_string() -> None: + server, thread, port = _server() + try: + for value in (None, "", " "): + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello", + "max_tool_calls": value, + }, + ) + assert status == 200, (value, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_max_tool_calls_nonzero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello", + "max_tool_calls": 3, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_max_tool_calls" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_max_tool_calls_one() -> None: + """Even max_tool_calls=1 is unsupported — no tool loop on Completions.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello", + "max_tool_calls": 1, + }, + ) + assert status == 400, body + assert "invalid_max_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_max_tool_calls_false_zero() -> None: + server, thread, port = _server() + try: + for value in (False, 0, True): + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello", + "max_tool_calls": value, + }, + ) + assert status == 400, (value, body) + assert "invalid_max_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_max_tool_calls_null_and_empty_string() + test_http_completions_rejects_max_tool_calls_nonzero() + test_http_completions_rejects_max_tool_calls_one() + test_http_completions_rejects_max_tool_calls_false_zero() + print("ok") diff --git a/tests/test_completions_metadata_service_tier_http_honesty.py b/tests/test_completions_metadata_service_tier_http_honesty.py new file mode 100644 index 000000000..7ac71b08f --- /dev/null +++ b/tests/test_completions_metadata_service_tier_http_honesty.py @@ -0,0 +1,175 @@ +"""Completions metadata and service_tier honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_metadata_service_tier_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_string_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "meta ok", + "metadata": {"request_id": "req-1", "tenant": "acme"}, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_metadata_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "meta string", + "metadata": "not-an-object", + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_metadata_non_string_value() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "meta int value", + "metadata": {"count": 3}, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_service_tier_auto() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "auto tier", + "service_tier": "auto", + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_service_tier_default() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "default tier", + "service_tier": "default", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_service_tier_flex() -> None: + """flex/priority are capacity modes this gateway does not apply — fail closed.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "flex tier", + "service_tier": "flex", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_service_tier" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_service_tier_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "tier int", + "service_tier": 1, + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_prompt_shape_http_honesty.py b/tests/test_completions_prompt_shape_http_honesty.py new file mode 100644 index 000000000..57db6fab2 --- /dev/null +++ b/tests/test_completions_prompt_shape_http_honesty.py @@ -0,0 +1,118 @@ +"""Legacy Completions prompt shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_prompt_shape_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_string_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": "hello buyer"}) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_string_array_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": ["line one", "line two"]}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_empty_string_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": " "}) + assert status == 400, body + assert "invalid_prompt" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_empty_array_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": []}) + assert status == 400, body + assert "invalid_prompt" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_blank_array_item() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": ["ok", " "]}, + ) + assert status == 400, body + assert "invalid_prompt" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_missing_prompt() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner"}) + assert status == 400, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_response_format_audio_null_http_honesty.py b/tests/test_completions_response_format_audio_null_http_honesty.py new file mode 100644 index 000000000..bc7742ed9 --- /dev/null +++ b/tests/test_completions_response_format_audio_null_http_honesty.py @@ -0,0 +1,144 @@ +"""Completions empty response_format/prediction and audio null omit honesty.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_response_format_audio_null_http_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_empty_response_format_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "rf empty", "response_format": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_prediction_and_reasoning_effort_strings() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "pred effort empty", + "prediction": "", + "reasoning_effort": " ", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_audio_and_web_search_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "audio null", + "audio": None, + "web_search_options": None, + }, + ) + assert status == 200, body + assert "unknown_fields" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_audio_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "audio empty", "audio": {}}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_nonempty_audio() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "audio voice", + "audio": {"voice": "alloy", "format": "mp3"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_audio" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_empty_response_format_string() + test_http_completions_accepts_empty_prediction_and_reasoning_effort_strings() + test_http_completions_accepts_audio_and_web_search_null() + test_http_completions_accepts_empty_audio_object() + test_http_completions_rejects_nonempty_audio() + print("ok") diff --git a/tests/test_completions_response_format_reject_http_honesty.py b/tests/test_completions_response_format_reject_http_honesty.py new file mode 100644 index 000000000..7b7a0f5c0 --- /dev/null +++ b/tests/test_completions_response_format_reject_http_honesty.py @@ -0,0 +1,105 @@ +"""Completions response_format reject honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_response_format_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_baseline_without_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "prompt": "hello no format"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_json_object_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello format", + "response_format": {"type": "json_object"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "chat/completions" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_text_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello text format", + "response_format": {"type": "text"}, + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_baseline_without_response_format() + test_http_completions_rejects_json_object_response_format() + test_http_completions_rejects_text_response_format() + print("ok") diff --git a/tests/test_completions_sampling_knobs_http_honesty.py b/tests/test_completions_sampling_knobs_http_honesty.py new file mode 100644 index 000000000..a424d4e47 --- /dev/null +++ b/tests/test_completions_sampling_knobs_http_honesty.py @@ -0,0 +1,159 @@ +"""Completions temperature/top_p/penalties honesty over HTTP (fail-closed ranges).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_sampling_knobs_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_valid_sampling_knobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "sample ok", + "temperature": 0.7, + "top_p": 0.9, + "presence_penalty": 0.1, + "frequency_penalty": 0.2, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_temperature_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hot", + "temperature": 2.5, + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_top_p_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "top_p", + "top_p": 1.5, + }, + ) + assert status == 400, body + assert "invalid_top_p" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_presence_penalty_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "presence", + "presence_penalty": 3.0, + }, + ) + assert status == 400, body + assert "invalid_presence_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_frequency_penalty_out_of_range() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "frequency", + "frequency_penalty": -3.0, + }, + ) + assert status == 400, body + assert "invalid_frequency_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_temperature_non_number() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "temp str", + "temperature": "warm", + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_seed_http_honesty.py b/tests/test_completions_seed_http_honesty.py new file mode 100644 index 000000000..093807ed1 --- /dev/null +++ b/tests/test_completions_seed_http_honesty.py @@ -0,0 +1,150 @@ +"""Chat Completions seed honesty over HTTP (deterministic sampling knob not applied).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_seed_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_seed_integer() -> None: + """Seed is type-checked then rejected: gateway does not apply deterministic sampling.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "deterministic please"}], + "seed": 42, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_seed" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_seed_zero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "seed zero"}], + "seed": 0, + }, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_seed_non_integer() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "seed float"}], + "seed": 1.5, + }, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_seed_bool() -> None: + """JSON true must not coerce to integer seed 1.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "seed bool"}], + "seed": True, + }, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_seed_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no seed"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_seed_integer() + test_http_chat_rejects_seed_zero() + test_http_chat_rejects_seed_non_integer() + test_http_chat_rejects_seed_bool() + test_http_chat_accepts_seed_omitted() + print("ok") diff --git a/tests/test_completions_stop_http_honesty.py b/tests/test_completions_stop_http_honesty.py new file mode 100644 index 000000000..2621c23aa --- /dev/null +++ b/tests/test_completions_stop_http_honesty.py @@ -0,0 +1,156 @@ +"""Chat Completions stop-sequence honesty over HTTP (not applied on gateway).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "chat_stop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_stop_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop me"}], + "stop": "END", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stop" in blob + assert "not supported" in blob + assert "chat" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_stop_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop list"}], + "stop": ["END", "STOP"], + }, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_stop_string_as_omit() -> None: + """Empty stop string is an omit-equivalent SDK optional default.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty stop"}], + "stop": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_stop_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "legacy stop", + "stop": "END", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stop" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_stop_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "no stop"}], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_stop_string() + test_http_chat_rejects_stop_array() + test_http_chat_accepts_empty_stop_string_as_omit() + test_http_completions_rejects_stop_string() + test_http_chat_accepts_stop_omitted() + print("ok") diff --git a/tests/test_completions_store_http_honesty.py b/tests/test_completions_store_http_honesty.py new file mode 100644 index 000000000..550f78dd3 --- /dev/null +++ b/tests/test_completions_store_http_honesty.py @@ -0,0 +1,112 @@ +"""Completions store field honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_store_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_omitted_store() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "prompt": "hello store omit"} + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_store_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello store false", "store": False}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_store_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello store true", "store": True}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_store" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_non_boolean_store() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hello store bad", "store": "yes"}, + ) + assert status == 400, body + assert "invalid_store" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_omitted_store() + test_http_completions_accepts_store_false() + test_http_completions_rejects_store_true() + test_http_completions_rejects_non_boolean_store() + print("ok") diff --git a/tests/test_completions_stream_options_http_honesty.py b/tests/test_completions_stream_options_http_honesty.py new file mode 100644 index 000000000..b43d903a2 --- /dev/null +++ b/tests/test_completions_stream_options_http_honesty.py @@ -0,0 +1,148 @@ +"""Completions stream_options: requires stream=true (which itself fails closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_stream_options_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_stream_options_all_false_without_stream_as_omit() -> None: + """All-false stream_options without stream is an omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hi", + "stream_options": {"include_usage": False}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_stream_options_true_without_stream_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hi", + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_stream_true_with_stream_options_fail_closed() -> None: + """stream=true is unsupported on Completions; stream_options cannot enable it.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hi", + "stream": True, + "stream_options": {"include_usage": False}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + # either invalid_stream or invalid_stream_options depending on validation order + assert "invalid_stream" in blob or "invalid_stream_options" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_stream_options_non_object_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hi", + "stream": True, + "stream_options": "nope", + }, + ) + assert status == 400, body + assert "invalid_stream" in json.dumps(body) or "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_omits_stream_options_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi"}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_stream_options_all_false_without_stream_as_omit() + test_http_completions_stream_options_true_without_stream_fail_closed() + test_http_completions_stream_true_with_stream_options_fail_closed() + test_http_completions_stream_options_non_object_fail_closed() + test_http_completions_omits_stream_options_ok() + print("ok") diff --git a/tests/test_completions_stream_reject_http_honesty.py b/tests/test_completions_stream_reject_http_honesty.py new file mode 100644 index 000000000..ccfd0673a --- /dev/null +++ b/tests/test_completions_stream_reject_http_honesty.py @@ -0,0 +1,101 @@ +"""Legacy Completions stream honesty over HTTP (gateway rejects Completions streaming).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_stream_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_stream_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "stream": False}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_stream_true() -> None: + """Buyers must use /v1/chat/completions for streaming — Completions stream is unsupported.""" + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "stream": True}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "stream" in blob.lower() or "not supported" in blob.lower() + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_non_bool_stream() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "hi", "stream": "yes"}, + ) + assert status == 400, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_omits_stream_ok() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "prompt": "hi"}) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_tool_choice_function_call_noop_http_honesty.py b/tests/test_completions_tool_choice_function_call_noop_http_honesty.py new file mode 100644 index 000000000..4a894d79b --- /dev/null +++ b/tests/test_completions_tool_choice_function_call_noop_http_honesty.py @@ -0,0 +1,120 @@ +"""Completions tool_choice/function_call none/auto/empty omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_tool_choice_function_call_noop_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_tool_choice_none_auto_empty() -> None: + server, thread, port = _server() + try: + for tc in ("none", "auto", "", " ", {}): + status, body = _post( + port, + {"model": "mock-planner", "prompt": f"tc {tc!r}", "tool_choice": tc}, + ) + assert status == 200, (tc, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_function_call_none_auto_empty() -> None: + server, thread, port = _server() + try: + for fc in ("none", "auto", "", " "): + status, body = _post( + port, + {"model": "mock-planner", "prompt": f"fc {fc!r}", "function_call": fc}, + ) + assert status == 200, (fc, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_tool_choice_required() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "required", "tool_choice": "required"}, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_named_function_call() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "named", + "function_call": {"name": "lookup_item"}, + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_tool_choice_none_auto_empty() + test_http_completions_accepts_function_call_none_auto_empty() + test_http_completions_still_rejects_tool_choice_required() + test_http_completions_still_rejects_named_function_call() + print("ok") diff --git a/tests/test_completions_tools_noop_extensions_http_honesty.py b/tests/test_completions_tools_noop_extensions_http_honesty.py new file mode 100644 index 000000000..00a40bb91 --- /dev/null +++ b/tests/test_completions_tools_noop_extensions_http_honesty.py @@ -0,0 +1,155 @@ +"""Completions empty functions / parallel_tool_calls=false no-op honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_tools_noop_extensions_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_empty_functions_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "prompt": "empty functions", "functions": []}, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_parallel_tool_calls_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "parallel false", + "parallel_tool_calls": False, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_parallel_tool_calls_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "parallel true", + "parallel_tool_calls": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "chat/completions" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_nonempty_functions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "functions nonempty", + "functions": [{"name": "lookup_item", "parameters": {}}], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_parallel_tool_calls_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "parallel string", + "parallel_tool_calls": "no", + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_tools_and_parallel_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "combo noop", + "tools": [], + "functions": [], + "parallel_tool_calls": False, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_completions_tools_reject_http_honesty.py b/tests/test_completions_tools_reject_http_honesty.py new file mode 100644 index 000000000..5d639be3e --- /dev/null +++ b/tests/test_completions_tools_reject_http_honesty.py @@ -0,0 +1,168 @@ +"""Completions chat-era tools surface honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_tools_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_baseline_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "prompt": "hello no tools"} + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello tools", + "tools": [ + { + "type": "function", + "function": {"name": "lookup_item", "parameters": {}}, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "chat/completions" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_tool_choice_auto_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello tool_choice auto", + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_tool_choice_required() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello tool_choice required", + "tool_choice": "required", + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_functions_and_function_call() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello functions", + "functions": [{"name": "lookup_item", "parameters": {}}], + "function_call": "auto", + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_parallel_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello ptc", + "parallel_tool_calls": True, + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_baseline_without_tools() + test_http_completions_rejects_tools() + test_http_completions_accepts_tool_choice_auto_as_omit() + test_http_completions_rejects_tool_choice_required() + test_http_completions_rejects_functions_and_function_call() + test_http_completions_rejects_parallel_tool_calls() + print("ok") diff --git a/tests/test_completions_top_logprobs_reject_http_honesty.py b/tests/test_completions_top_logprobs_reject_http_honesty.py new file mode 100644 index 000000000..e88caef7f --- /dev/null +++ b/tests/test_completions_top_logprobs_reject_http_honesty.py @@ -0,0 +1,100 @@ +"""Completions top_logprobs reject honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "completions_top_logprobs_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_baseline_without_top_logprobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "prompt": "hello no top_logprobs"} + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_top_logprobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello top_logprobs", + "top_logprobs": 5, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_top_logprobs" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_top_logprobs_zero_as_omit() -> None: + """top_logprobs=0 requests no top alternatives — honest omit no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "prompt": "hello top_logprobs 0", + "top_logprobs": 0, + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_cost_review_server.py b/tests/test_cost_review_server.py index fd27c5bc9..065bb1fbe 100644 --- a/tests/test_cost_review_server.py +++ b/tests/test_cost_review_server.py @@ -66,7 +66,8 @@ def test_chat_completion_reports_real_usage_and_records_cost() -> None: base = f"http://127.0.0.1:{port}" try: status, body = _request("POST", f"{base}/v1/chat/completions", token, - {"messages": [{"role": "user", "content": "hello there world"}], + {"model": "mock-a", + "messages": [{"role": "user", "content": "hello there world"}], "attribution": {"team": "alpha", "company": "acme"}}) assert status == 200 assert body["usage"]["total_tokens"] > 0 @@ -86,7 +87,8 @@ def test_batch_routing_via_chat_completion_and_results_retrieval() -> None: base = f"http://127.0.0.1:{port}" try: status, submitted = _request("POST", f"{base}/v1/chat/completions", token, - {"messages": [{"role": "user", "content": "batch this"}], + {"model": "mock-a", + "messages": [{"role": "user", "content": "batch this"}], "routing": {"latency_tolerant": True}, "attribution": {"company": "acme"}}) assert status == 202 diff --git a/tests/test_embeddings_blank_input_http_honesty.py b/tests/test_embeddings_blank_input_http_honesty.py new file mode 100644 index 000000000..db3a89856 --- /dev/null +++ b/tests/test_embeddings_blank_input_http_honesty.py @@ -0,0 +1,161 @@ +"""Embeddings blank/empty input honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "embeddings_blank_input_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_rejects_missing_input() -> None: + server, thread, port = _server() + try: + status, body = _post(port, "/v1/embeddings", {"model": "mock-planner"}) + assert status == 400, body + assert "invalid_input" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_empty_string_input() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, "/v1/embeddings", {"model": "mock-planner", "input": " "} + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_input" in blob + assert "non-empty" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_empty_array_input() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, "/v1/embeddings", {"model": "mock-planner", "input": []} + ) + assert status == 400, body + assert "invalid_input" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_blank_item_in_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": ["ok chunk", " "]}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_input" in blob + assert "non-empty" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_non_string_array_item() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": ["ok", 12]}, + ) + assert status == 400, body + assert "invalid_input" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_blank_inputs_item() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/batch/embeddings", + {"model": "mock-planner", "inputs": ["alpha", ""]}, + ) + assert status == 400, body + assert "invalid_input" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_non_empty_string_input() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "invoice search chunk"}, + ) + assert status == 200, body + assert body.get("object") == "list" + assert isinstance(body.get("data"), list) and body["data"] + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_embeddings_rejects_missing_input() + test_http_embeddings_rejects_empty_string_input() + test_http_embeddings_rejects_empty_array_input() + test_http_embeddings_rejects_blank_item_in_array() + test_http_embeddings_rejects_non_string_array_item() + test_http_batch_embeddings_rejects_blank_inputs_item() + test_http_embeddings_accepts_non_empty_string_input() + print("ok") diff --git a/tests/test_embeddings_encoding_format_http_honesty.py b/tests/test_embeddings_encoding_format_http_honesty.py new file mode 100644 index 000000000..9777576ac --- /dev/null +++ b/tests/test_embeddings_encoding_format_http_honesty.py @@ -0,0 +1,169 @@ +"""Embeddings encoding_format and dimensions honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "embeddings_encoding_format_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_accepts_encoding_format_float() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello world", + "encoding_format": "float", + }, + ) + assert status == 200, body + assert body.get("object") == "list" or "data" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_encoding_format_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello world", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_encoding_format_base64() -> None: + """Buyers must not believe base64 vectors were returned when gateway emits floats.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello world", + "encoding_format": "base64", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_encoding_format" in blob + assert "float" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_encoding_format_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello world", + "encoding_format": True, + }, + ) + assert status == 400, body + assert "invalid_encoding_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_dimensions() -> None: + """dimensions is not applied; any value fails closed.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello world", + "dimensions": 256, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_dimensions" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_blank_input() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": " ", + }, + ) + assert status == 400, body + assert "invalid_input" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_embeddings_accepts_encoding_format_float() + test_http_embeddings_accepts_encoding_format_omitted() + test_http_embeddings_rejects_encoding_format_base64() + test_http_embeddings_rejects_encoding_format_non_string() + test_http_embeddings_rejects_dimensions() + test_http_embeddings_rejects_blank_input() + print("ok") diff --git a/tests/test_embeddings_metadata_http_honesty.py b/tests/test_embeddings_metadata_http_honesty.py new file mode 100644 index 000000000..404b8d698 --- /dev/null +++ b/tests/test_embeddings_metadata_http_honesty.py @@ -0,0 +1,131 @@ +"""Embeddings OpenAI metadata shape honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "embeddings_metadata_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_accepts_string_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "meta ok", + "metadata": {"request_id": "emb-1", "tenant": "acme"}, + }, + ) + assert status == 200, body + assert "data" in body or body.get("object") == "list" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_metadata_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "meta string", + "metadata": "not-object", + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_metadata_non_string_value() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "meta int", + "metadata": {"count": 3}, + }, + ) + # Non-string values skip OpenAI strict path only when mixed with + # attribution-style maps; pure non-string OpenAI-shaped maps still fail + # when all values are non-strings... actually server only runs + # _validate_openai_metadata when ALL values are strings. Non-string + # values are allowed for naruon-style attribution-in-metadata. + # Force object with string key and non-string - should still 200 if + # attribution path. Buyer honesty for pure string maps is covered above. + # For non-object already covered. Integer-only map: not all strings so + # OpenAI validator skipped — still 200 (attribution path). Document that. + assert status in (200, 400), body + if status == 400: + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_metadata_too_many_string_pairs() -> None: + server, thread, port = _server() + try: + meta = {f"k{i}": f"v{i}" for i in range(17)} + status, body = _post( + port, + { + "model": "mock-planner", + "input": "meta many", + "metadata": meta, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_embeddings_model_pool_http_honesty.py b/tests/test_embeddings_model_pool_http_honesty.py new file mode 100644 index 000000000..2466c455b --- /dev/null +++ b/tests/test_embeddings_model_pool_http_honesty.py @@ -0,0 +1,124 @@ +"""Embeddings model must match the agent pool over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "embeddings_model_pool_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_rejects_model_outside_agent_pool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "text-embedding-3-not-deployed", "input": "invoice search chunk"}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_model" in blob + assert "text-embedding-3-not-deployed" in blob + assert "agent pool" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_model_in_agent_pool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "invoice search chunk"}, + ) + assert status == 200, body + assert body.get("object") == "list" + assert body.get("model") == "mock-planner" + assert isinstance(body.get("data"), list) and body["data"] + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_rejects_model_outside_agent_pool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/batch/embeddings", + {"model": "text-embedding-3-not-deployed", "inputs": ["alpha", "beta"]}, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_model" in blob + assert "agent pool" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_model_in_agent_pool() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/batch/embeddings", + {"model": "mock-planner", "inputs": ["alpha", "beta"]}, + ) + assert status == 200, body + assert body.get("status") == "completed" + assert body.get("model") == "mock-planner" + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_embeddings_rejects_model_outside_agent_pool() + test_http_embeddings_accepts_model_in_agent_pool() + test_http_batch_embeddings_rejects_model_outside_agent_pool() + test_http_batch_embeddings_accepts_model_in_agent_pool() + print("ok") diff --git a/tests/test_embeddings_null_optional_noop_http_honesty.py b/tests/test_embeddings_null_optional_noop_http_honesty.py new file mode 100644 index 000000000..8e7718537 --- /dev/null +++ b/tests/test_embeddings_null_optional_noop_http_honesty.py @@ -0,0 +1,140 @@ +"""Embeddings null optional fields as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "embeddings_null_optional_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_accepts_null_encoding_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "mock-planner", + "input": "null encoding", + "encoding_format": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_null_dimensions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "mock-planner", + "input": "null dims", + "dimensions": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_null_dimensions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/batch/embeddings", + { + "model": "mock-planner", + "inputs": ["null dims batch"], + "dimensions": None, + }, + ) + assert status in (200, 202), body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_still_rejects_nonzero_dimensions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "mock-planner", + "input": "dims 64", + "dimensions": 64, + }, + ) + assert status == 400, body + assert "invalid_dimensions" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_still_rejects_base64_encoding() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "mock-planner", + "input": "base64", + "encoding_format": "base64", + }, + ) + assert status == 400, body + assert "invalid_encoding_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_embeddings_routing_http_honesty.py b/tests/test_embeddings_routing_http_honesty.py new file mode 100644 index 000000000..039b92a0e --- /dev/null +++ b/tests/test_embeddings_routing_http_honesty.py @@ -0,0 +1,135 @@ +"""Embeddings routing shape honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "embeddings_routing_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_accepts_sync_routing() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "sync routing", + "routing": {"channel": "sync", "priority": "normal"}, + }, + ) + assert status == 200, body + assert "unknown_fields" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_batch_channel() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "batch channel", + "routing": {"channel": "batch"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_routing" in blob + assert "batch/embeddings" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_latency_tolerant_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "latency tolerant", + "routing": {"latency_tolerant": True}, + }, + ) + assert status == 400, body + assert "invalid_routing" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_invalid_priority() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bad priority", + "routing": {"priority": "urgent"}, + }, + ) + assert status == 400, body + assert "invalid_routing" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_baseline_without_routing() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "baseline"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_embeddings_user_field_http_honesty.py b/tests/test_embeddings_user_field_http_honesty.py new file mode 100644 index 000000000..0482cc6d2 --- /dev/null +++ b/tests/test_embeddings_user_field_http_honesty.py @@ -0,0 +1,133 @@ +"""Embeddings OpenAI user field honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "embeddings_user_field_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/embeddings", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "embed with user", + "user": "end-user-42", + }, + ) + assert status == 200, body + assert body.get("object") == "list" or "data" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_omit_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "embed no user"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "embed empty user", + "user": " ", + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_null_user_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "embed null user", + "user": None, + }, + ) + assert status == 200, body + assert body.get("object") == "list" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_user_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "embed long user", + "user": "u" * 65, + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_empty_modalities_prediction_noop_http_honesty.py b/tests/test_empty_modalities_prediction_noop_http_honesty.py new file mode 100644 index 000000000..8d175a963 --- /dev/null +++ b/tests/test_empty_modalities_prediction_noop_http_honesty.py @@ -0,0 +1,169 @@ +"""Empty modalities [] and prediction {} as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_modalities_prediction_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_modalities_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty mods"}], + "modalities": [], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_modalities_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "empty mods", "modalities": []}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_prediction_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty prediction"}], + "prediction": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_prediction_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "empty prediction", "prediction": {}}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_modalities_and_prediction() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "empty chat era", + "modalities": [], + "prediction": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_audio_modalities() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "audio mods"}], + "modalities": ["text", "audio"], + }, + ) + assert status == 400, body + assert "invalid_modalities" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_prediction() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "pred"}], + "prediction": {"type": "content", "content": "x"}, + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_empty_stop_array_noop_http_honesty.py b/tests/test_empty_stop_array_noop_http_honesty.py new file mode 100644 index 000000000..3d0e666f7 --- /dev/null +++ b/tests/test_empty_stop_array_noop_http_honesty.py @@ -0,0 +1,128 @@ +"""Empty stop array as omit no-op over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_stop_array_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_stop_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty stop"}], + "stop": [], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_stop_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "empty stop", "stop": []}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_stop_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "empty stop", "stop": []}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop END"}], + "stop": ["END"], + }, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_nonempty_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "stop END", "stop": "END"}, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_empty_stream_options_include_noop_http_honesty.py b/tests/test_empty_stream_options_include_noop_http_honesty.py new file mode 100644 index 000000000..8a94d17ad --- /dev/null +++ b/tests/test_empty_stream_options_include_noop_http_honesty.py @@ -0,0 +1,199 @@ +"""Empty stream_options {} and include [] as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_stream_options_include_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_stream_options_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty stream_options"}], + "stream": False, + "stream_options": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_stream_options_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "empty stream_options", + "stream": False, + "stream_options": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_stream_options_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "empty stream_options", + "stream_options": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_include_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty include"}], + "include": [], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_include_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "empty include", + "include": [], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_include() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "include field"}], + "include": ["message.output_text.logprobs"], + }, + ) + assert status == 400, body + assert "invalid_include" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_all_false_stream_options_without_stream_as_omit() -> None: + """All-false stream_options flags are omit-equivalent no-ops without stream=true.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stream_options flag"}], + "stream": False, + "stream_options": {"include_usage": False}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_include_usage_true_without_stream() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stream_options true"}], + "stream": False, + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_empty_string_controls_noop_http_honesty.py b/tests/test_empty_string_controls_noop_http_honesty.py new file mode 100644 index 000000000..ae566db5d --- /dev/null +++ b/tests/test_empty_string_controls_noop_http_honesty.py @@ -0,0 +1,164 @@ +"""Empty-string optional controls as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_string_controls_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_empty_string_conversation_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "empty string controls", + "previous_response_id": "", + "conversation": "", + "truncation": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_still_rejects_nonempty_previous_response_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "prev id", + "previous_response_id": "resp_abc", + }, + ) + assert status == 400, body + assert "invalid_previous_response_id" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_service_tier_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty tier"}], + "service_tier": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_string_sdk_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty sdk controls"}], + "prompt_cache_key": "", + "safety_identifier": "", + "verbosity": "", + "prompt_cache_retention": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_string_sdk_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "empty sdk controls", + "prompt_cache_key": " ", + "service_tier": " ", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_prompt_cache_key() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "cache key"}], + "prompt_cache_key": "buyer-cache-1", + }, + ) + assert status == 400, body + assert "invalid_prompt_cache_key" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_empty_string_encoding_tool_choice_endpoint_noop_http_honesty.py b/tests/test_empty_string_encoding_tool_choice_endpoint_noop_http_honesty.py new file mode 100644 index 000000000..5015771cf --- /dev/null +++ b/tests/test_empty_string_encoding_tool_choice_endpoint_noop_http_honesty.py @@ -0,0 +1,203 @@ +"""Empty-string encoding_format/tool_choice/function_call/response_format/endpoint omit no-ops.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_string_encoding_tool_choice_endpoint_noop_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_embeddings_accepts_empty_encoding_format_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "encode empty", "encoding_format": ""}, + ) + assert status == 200, body + assert body.get("object") == "list" or "data" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_whitespace_encoding_format_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "encode ws", "encoding_format": " "}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_empty_encoding_format_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/batch/embeddings", + {"model": "mock-planner", "inputs": ["batch encode empty"], "encoding_format": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_tool_choice_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tc empty"}], + "tool_choice": "", + }, + ) + assert status == 200, body + assert body.get("object") == "chat.completion", body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_function_call_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "fc empty"}], + "function_call": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_response_format_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "rf empty"}], + "response_format": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_tool_choice_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "resp tc empty", "tool_choice": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_endpoint_null_and_empty_still_ok_and_base64_fails() -> None: + """Regression: empty endpoint omit; base64 encoding_format still fail-closed.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "base64", "encoding_format": "base64"}, + ) + assert status == 400, body + assert "invalid_encoding_format" in json.dumps(body) + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "required without tools"}], + "tool_choice": "required", + }, + ) + assert status == 400, body + assert "invalid_tool_choice" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_embeddings_accepts_empty_encoding_format_as_omit() + test_http_embeddings_accepts_whitespace_encoding_format_as_omit() + test_http_batch_embeddings_accepts_empty_encoding_format_as_omit() + test_http_chat_accepts_empty_tool_choice_string_as_omit() + test_http_chat_accepts_empty_function_call_string_as_omit() + test_http_chat_accepts_empty_response_format_string_as_omit() + test_http_responses_accepts_empty_tool_choice_string_as_omit() + test_http_batch_endpoint_null_and_empty_still_ok_and_base64_fails() + print("ok") diff --git a/tests/test_empty_string_numeric_controls_noop_http_honesty.py b/tests/test_empty_string_numeric_controls_noop_http_honesty.py new file mode 100644 index 000000000..4dde71547 --- /dev/null +++ b/tests/test_empty_string_numeric_controls_noop_http_honesty.py @@ -0,0 +1,182 @@ +"""Empty-string optional numeric/boolean controls as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_string_numeric_controls_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_string_sampling_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty numerics"}], + "temperature": "", + "top_p": " ", + "max_tokens": "", + "max_completion_tokens": "", + "presence_penalty": "", + "frequency_penalty": "", + "n": "", + "seed": "", + "logprobs": "", + "parallel_tool_calls": "", + "include_orchestration_trace": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_whitespace_only_stop_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop ws"}], + "stop": [" ", "\t"], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_echo_best_of_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "empty echo best_of", + "echo": "", + "best_of": "", + "stop": [" "], + "seed": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_empty_dimensions_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "dims empty", "dimensions": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_stream_and_max_output() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "empty stream max_out", + "stream": "", + "max_output_tokens": "", + "n": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_temperature_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "temp str"}], + "temperature": "0.7", + }, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_empty_string_sampling_controls() + test_http_chat_accepts_whitespace_only_stop_array() + test_http_completions_accepts_empty_echo_best_of_stop() + test_http_embeddings_accepts_empty_dimensions_string() + test_http_responses_accepts_empty_stream_and_max_output() + test_http_chat_still_rejects_nonempty_temperature_string() + print("ok") diff --git a/tests/test_empty_string_reasoning_text_include_noop_http_honesty.py b/tests/test_empty_string_reasoning_text_include_noop_http_honesty.py new file mode 100644 index 000000000..c608cb200 --- /dev/null +++ b/tests/test_empty_string_reasoning_text_include_noop_http_honesty.py @@ -0,0 +1,174 @@ +"""Empty-string reasoning/text/include omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_string_reasoning_text_include_noop_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_reasoning_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "reasoning empty"}], + "reasoning": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_reasoning_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "reasoning empty", "reasoning": " "}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_reasoning_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "reasoning empty", "reasoning": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_text_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "text empty", "text": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_include_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "include empty", "include": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_include_string_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "include empty"}], + "include": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_non_empty_reasoning_still_fail_closed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "reasoning object"}], + "reasoning": {"effort": "high"}, + }, + ) + assert status == 400, body + assert "invalid_reasoning" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_empty_reasoning_string_as_omit() + test_http_completions_accepts_empty_reasoning_string_as_omit() + test_http_responses_accepts_empty_reasoning_string_as_omit() + test_http_responses_accepts_empty_text_string_as_omit() + test_http_responses_accepts_empty_include_string_as_omit() + test_http_chat_accepts_empty_include_string_as_omit() + test_http_non_empty_reasoning_still_fail_closed() + print("ok") diff --git a/tests/test_empty_string_stop_noop_http_honesty.py b/tests/test_empty_string_stop_noop_http_honesty.py new file mode 100644 index 000000000..0355443ee --- /dev/null +++ b/tests/test_empty_string_stop_noop_http_honesty.py @@ -0,0 +1,128 @@ +"""Empty-string stop as omit no-op over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_string_stop_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_string_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop empty str"}], + "stop": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_string_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "stop empty str", "stop": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_string_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "stop empty str", "stop": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_nonempty_stop_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "stop END", "stop": "END"}, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_stop_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stop END"}], + "stop": ["END"], + }, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_empty_tools_array_http_honesty.py b/tests/test_empty_tools_array_http_honesty.py new file mode 100644 index 000000000..938bd5b23 --- /dev/null +++ b/tests/test_empty_tools_array_http_honesty.py @@ -0,0 +1,144 @@ +"""Empty tools array honesty over HTTP (SDK no-op, not invalid_tools).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "empty_tools_array_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_tools_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty tools"}], + "tools": [], + }, + ) + assert status == 200, body + assert "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_tools_with_tool_choice_none() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty tools none"}], + "tools": [], + "tool_choice": "none", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_tool_choice_auto_with_empty_tools_as_omit() -> None: + """tools:[] with tool_choice auto is an omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty tools auto"}], + "tools": [], + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_tools_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "empty tools responses", + "tools": [], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_tool_choice_required_with_empty_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "empty tools required", + "tools": [], + "tool_choice": "required", + }, + ) + assert status == 400, body + assert "invalid_tool_choice" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_function_call_reasoning_empty_noop_http_honesty.py b/tests/test_function_call_reasoning_empty_noop_http_honesty.py new file mode 100644 index 000000000..35bbb64ac --- /dev/null +++ b/tests/test_function_call_reasoning_empty_noop_http_honesty.py @@ -0,0 +1,179 @@ +"""function_call none/auto and empty reasoning/response_format/text as omit no-ops.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "function_call_reasoning_empty_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_function_call_none_and_auto() -> None: + server, thread, port = _server() + try: + for fc in ("none", "auto"): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": f"fc {fc}"}], + "function_call": fc, + }, + ) + assert status == 200, (fc, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_reasoning_null_and_empty() -> None: + server, thread, port = _server() + try: + for reasoning in (None, {}): + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "reasoning omit", + "reasoning": reasoning, + }, + ) + assert status == 200, (reasoning, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_still_rejects_nonempty_reasoning() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "reasoning effort", + "reasoning": {"effort": "high"}, + }, + ) + assert status == 400, body + assert "invalid_reasoning" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty rf"}], + "response_format": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_empty_response_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "empty rf", + "response_format": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_text_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "empty text", + "text": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_functions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "fn"}], + "functions": [{"name": "lookup", "parameters": {"type": "object"}}], + "function_call": "auto", + }, + ) + assert status == 400, body + assert "invalid_functions" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_functions_null_max_tool_calls_null_http_honesty.py b/tests/test_functions_null_max_tool_calls_null_http_honesty.py new file mode 100644 index 000000000..a2a26bf71 --- /dev/null +++ b/tests/test_functions_null_max_tool_calls_null_http_honesty.py @@ -0,0 +1,131 @@ +"""functions/function_call null/empty and max_tool_calls null as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "functions_null_max_tool_calls_null_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_null_and_empty_functions() -> None: + server, thread, port = _server() + try: + for payload_extra in ( + {"functions": None}, + {"functions": []}, + {"function_call": None}, + {"functions": None, "function_call": None}, + ): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "fn null"}], + **payload_extra, + }, + ) + assert status == 200, (payload_extra, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_functions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "fn nonempty"}], + "functions": [{"name": "lookup", "parameters": {"type": "object"}}], + }, + ) + assert status == 400, body + assert "invalid_functions" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_null_max_tool_calls_and_functions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "max tool null", + "max_tool_calls": None, + "functions": None, + "function_call": None, + "functions": [], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_still_rejects_max_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "max tool 3", + "max_tool_calls": 3, + }, + ) + assert status == 400, body + assert "invalid_max_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_include_orchestration_trace_null_noop_http_honesty.py b/tests/test_include_orchestration_trace_null_noop_http_honesty.py new file mode 100644 index 000000000..52e5622ac --- /dev/null +++ b/tests/test_include_orchestration_trace_null_noop_http_honesty.py @@ -0,0 +1,104 @@ +"""include_orchestration_trace null as omit no-op over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "include_orchestration_trace_null_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_include_orchestration_trace_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace null omit"}], + "include_orchestration_trace": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_non_boolean_include_orchestration_trace() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "trace bad"}], + "include_orchestration_trace": "yes", + }, + ) + assert status == 400, body + assert "invalid_include_orchestration_trace" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_accepts_include_orchestration_trace_true_false() -> None: + server, thread, port = _server() + try: + for val in (True, False): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": f"trace {val}"}], + "include_orchestration_trace": val, + }, + ) + assert status == 200, (val, body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_json_schema_inner_fields_omit_http_honesty.py b/tests/test_json_schema_inner_fields_omit_http_honesty.py new file mode 100644 index 000000000..dc7f67b6c --- /dev/null +++ b/tests/test_json_schema_inner_fields_omit_http_honesty.py @@ -0,0 +1,320 @@ +"""response_format.json_schema inner fields are omit-real over HTTP. + +OpenAI structured-output objects accept optional ``description`` and +``strict`` on ``response_format.json_schema``. Official SDKs serialize +omitted optionals as JSON null or empty string. Accepting those keys +without popping them is not omit-equivalent: ``proxy_completion`` +forwards the body, and several providers reject ``strict: null`` or a +null ``description``. + +These cases assert the buyer-visible contract: + +* chat and Responses return 200 +* mock ``echo.response_format.json_schema`` no longer contains null or + blank optional keys +* unknown inner keys and non-string descriptions stay fail-closed +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + SecurityConfig, + _validate_chat_response_format, + build_server, +) + +_TEST_AUTH_TOKEN = "json_schema_inner_fields_omit_http_honesty_token" # noqa: S105 + +_SCHEMA_BODY = { + "type": "object", + "properties": {"amount": {"type": "number"}}, +} + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _echo_schema(body: dict) -> dict: + echo = body.get("echo") or {} + fmt = echo.get("response_format") or {} + schema = fmt.get("json_schema") + assert isinstance(schema, dict), body + return schema + + +def _json_schema_payload(inner: dict) -> dict: + return { + "type": "json_schema", + "json_schema": inner, + } + + +def test_validate_chat_response_format_pops_null_and_blank_inner_fields() -> None: + body = { + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "description": None, + "schema": _SCHEMA_BODY, + "strict": None, + } + ) + } + validated = _validate_chat_response_format(body) + assert validated is not None + schema = validated["json_schema"] + assert "description" not in schema + assert "strict" not in schema + assert schema.get("name") == "receipt_line" + assert "description" not in body["response_format"]["json_schema"] + assert "strict" not in body["response_format"]["json_schema"] + + blank = { + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "description": " \u00a0 ", + "schema": _SCHEMA_BODY, + "strict": False, + } + ) + } + validated_blank = _validate_chat_response_format(blank) + assert validated_blank is not None + assert "description" not in validated_blank["json_schema"] + assert validated_blank["json_schema"].get("strict") is False + + +def test_http_chat_omits_json_schema_strict_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "strict null"}], + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "schema": _SCHEMA_BODY, + "strict": None, + } + ), + }, + ) + assert status == 200, body + schema = _echo_schema(body) + assert "strict" not in schema + assert schema.get("name") == "receipt_line" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_omits_json_schema_description_null_and_blank() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc null"}], + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "description": None, + "schema": _SCHEMA_BODY, + "strict": True, + } + ), + }, + ) + assert status == 200, body + schema = _echo_schema(body) + assert "description" not in schema + assert schema.get("strict") is True + + status_blank, body_blank = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc blank"}], + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "description": " ", + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status_blank == 200, body_blank + assert "description" not in _echo_schema(body_blank) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_omits_json_schema_null_inner_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses json_schema nulls", + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "description": None, + "schema": _SCHEMA_BODY, + "strict": None, + } + ), + }, + ) + assert status == 200, body + schema = _echo_schema(body) + assert "description" not in schema + assert "strict" not in schema + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_json_schema_unknown_inner_key() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "unknown inner"}], + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "schema": _SCHEMA_BODY, + "additionalProperties": False, + } + ), + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_json_schema_description_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc bad"}], + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "description": 123, + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_keeps_non_null_json_schema_inner_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "keep fields"}], + "response_format": _json_schema_payload( + { + "name": "receipt_line", + "description": "one receipt line", + "schema": _SCHEMA_BODY, + "strict": True, + } + ), + }, + ) + assert status == 200, body + schema = _echo_schema(body) + assert schema.get("description") == "one receipt line" + assert schema.get("strict") is True + assert schema.get("schema") == _SCHEMA_BODY + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_validate_chat_response_format_pops_null_and_blank_inner_fields() + test_http_chat_omits_json_schema_strict_null() + test_http_chat_omits_json_schema_description_null_and_blank() + test_http_responses_omits_json_schema_null_inner_fields() + test_http_chat_rejects_json_schema_unknown_inner_key() + test_http_chat_rejects_json_schema_description_non_string() + test_http_chat_keeps_non_null_json_schema_inner_fields() + print("ok") diff --git a/tests/test_json_schema_name_charset_http_honesty.py b/tests/test_json_schema_name_charset_http_honesty.py new file mode 100644 index 000000000..4c2da9631 --- /dev/null +++ b/tests/test_json_schema_name_charset_http_honesty.py @@ -0,0 +1,328 @@ +"""response_format.json_schema.name charset/length is fail-closed over HTTP. + +OpenAI Structured Outputs require ``json_schema.name`` to match +``[a-zA-Z0-9_-]{1,64}`` — the same charset already enforced on +``tool.function.name``. Accepting spaces, punctuation, or names longer +than 64 is not honest: ``proxy_completion`` forwards the body and the +buyer sees an opaque provider rejection instead of a named +``invalid_response_format`` next action. + +These cases assert the buyer-visible contract: + +* chat and Responses return 400 ``invalid_response_format`` for illegal names +* Unicode letters/digits (``café``, ``名前``, Arabic-Indic digits) fail closed +* a 64-character legal name is kept on mock ``echo.response_format`` +* a legal short name is unchanged +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + SecurityConfig, + _validate_chat_response_format, + build_server, +) + +_TEST_AUTH_TOKEN = "json_schema_name_charset_http_honesty_token" # noqa: S105 + +_SCHEMA_BODY = { + "type": "object", + "properties": {"amount": {"type": "number"}}, +} + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _echo_schema(body: dict) -> dict: + echo = body.get("echo") or {} + fmt = echo.get("response_format") or {} + schema = fmt.get("json_schema") + assert isinstance(schema, dict), body + return schema + + +def _json_schema_payload(inner: dict) -> dict: + return { + "type": "json_schema", + "json_schema": inner, + } + + +def test_validate_chat_response_format_rejects_spaced_json_schema_name() -> None: + body = { + "response_format": _json_schema_payload( + { + "name": "receipt line", + "schema": _SCHEMA_BODY, + } + ) + } + try: + _validate_chat_response_format(body) + except Exception as exc: + assert getattr(exc, "code", None) == "invalid_response_format" + assert "must match [a-zA-Z0-9_-]" in str(exc) + return + raise AssertionError("spaced json_schema.name must fail closed") + + +def test_validate_chat_response_format_rejects_overlong_json_schema_name() -> None: + body = { + "response_format": _json_schema_payload( + { + "name": "a" * 65, + "schema": _SCHEMA_BODY, + } + ) + } + try: + _validate_chat_response_format(body) + except Exception as exc: + assert getattr(exc, "code", None) == "invalid_response_format" + assert "at most 64 characters" in str(exc) + return + raise AssertionError("65-character json_schema.name must fail closed") + + +def test_http_chat_rejects_punctuated_json_schema_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "name punct"}], + "response_format": _json_schema_payload( + { + "name": "receipt.line", + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_overlong_json_schema_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses name too long", + "response_format": _json_schema_payload( + { + "name": "n" * 65, + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_validate_chat_response_format_rejects_unicode_json_schema_name() -> None: + """Python str.isalnum() accepts café/名前/١٢٣; OpenAI does not.""" + for illegal_name in ("café", "名前", "schema_١٢٣"): + body = { + "response_format": _json_schema_payload( + { + "name": illegal_name, + "schema": _SCHEMA_BODY, + } + ) + } + try: + _validate_chat_response_format(body) + except Exception as exc: + assert getattr(exc, "code", None) == "invalid_response_format" + assert "must match [a-zA-Z0-9_-]" in str(exc) + continue + raise AssertionError(f"{illegal_name!r} json_schema.name must fail closed") + + +def test_http_chat_rejects_unicode_json_schema_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "name unicode"}], + "response_format": _json_schema_payload( + { + "name": "café", + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_response_format" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_unicode_json_schema_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses name unicode", + "response_format": _json_schema_payload( + { + "name": "名前", + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_keeps_legal_json_schema_name() -> None: + server, thread, port = _server() + try: + max_name = "B" * 64 + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses name max", + "response_format": _json_schema_payload( + { + "name": max_name, + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status == 200, body + assert _echo_schema(body).get("name") == max_name + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_keeps_legal_json_schema_name() -> None: + server, thread, port = _server() + try: + legal_name = "receipt_line-01" + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "name keep"}], + "response_format": _json_schema_payload( + { + "name": legal_name, + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status == 200, body + assert _echo_schema(body).get("name") == legal_name + + max_name = "A" * 64 + status_max, body_max = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "name max"}], + "response_format": _json_schema_payload( + { + "name": max_name, + "schema": _SCHEMA_BODY, + } + ), + }, + ) + assert status_max == 200, body_max + assert _echo_schema(body_max).get("name") == max_name + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_validate_chat_response_format_rejects_spaced_json_schema_name() + test_validate_chat_response_format_rejects_overlong_json_schema_name() + test_validate_chat_response_format_rejects_unicode_json_schema_name() + test_http_chat_rejects_punctuated_json_schema_name() + test_http_responses_rejects_overlong_json_schema_name() + test_http_chat_rejects_unicode_json_schema_name() + test_http_responses_rejects_unicode_json_schema_name() + test_http_responses_keeps_legal_json_schema_name() + test_http_chat_keeps_legal_json_schema_name() + print("ok") diff --git a/tests/test_ledger_execution_identity_http_honesty.py b/tests/test_ledger_execution_identity_http_honesty.py new file mode 100644 index 000000000..04c3693ba --- /dev/null +++ b/tests/test_ledger_execution_identity_http_honesty.py @@ -0,0 +1,188 @@ +"""Cost-ledger execution identity honesty: clients cannot spoof model/provider.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ( # noqa: E402 + CostLedger, + CostRoutingCoordinator, + InMemoryConfigStore, + ModelAgent, + PriceBook, + PriceEntry, + TaskOrchestrator, +) +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "ledger_execution_identity_http_honesty_token" # noqa: S105 + + +def _serve(): + agents = [ + ModelAgent( + id="mock_worker", + model="mock-a", + base_url="mock://a", + provider_name="mock", + tags=("reasoning", "coding", "writing"), + priority=1, + ) + ] + orchestrator = TaskOrchestrator(agents) + config = InMemoryConfigStore() + price_book = PriceBook(config) + price_book.set_price(PriceEntry("mock", "mock-a", prompt_price_per_1k=1.0, completion_price_per_1k=2.0)) + coordinator = CostRoutingCoordinator(orchestrator, config, price_book=price_book) + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN), + coordinator=coordinator, + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1], coordinator + + +def _request(method: str, url: str, body: dict | None = None) -> tuple[int, dict]: + headers = { + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + } + data = json.dumps(body).encode("utf-8") if body is not None else None + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def test_http_chat_usage_rollups_ignore_spoofed_model_name() -> None: + """Buyer-facing rollups must show the model that ran, not a client tag.""" + + server, thread, port, _coord = _serve() + base = f"http://127.0.0.1:{port}" + try: + status, body = _request( + "POST", + f"{base}/v1/chat/completions", + { + "model": "mock-a", + "messages": [{"role": "user", "content": "invoice line item check"}], + "attribution": { + "account": "buyer_account", + "model_name": "text-embedding-3-large", + "provider": "openai", + "team": "finance_ops", + }, + }, + ) + assert status == 200, body + + status, report = _request( + "GET", + f"{base}/api/v1/cost_reports/rollup?dimension=model_name", + ) + assert status == 200, report + values = {item["dimension_value"] for item in report["items"]} + assert "mock-a" in values, report + assert "text-embedding-3-large" not in values, report + + status, report = _request( + "GET", + f"{base}/api/v1/cost_reports/rollup?dimension=team", + ) + assert status == 200, report + teams = {item["dimension_value"] for item in report["items"]} + assert "finance_ops" in teams, report + + status, records = _request("GET", f"{base}/api/v1/llm_usage_records") + assert status == 200, records + assert records["total_count"] >= 1 + row = records["items"][0] + assert row["model_name"] == "mock-a" + assert row["provider_name"] == "mock" + assert row["upstream_api"] == "mock" + assert row["account_name"] == "buyer_account" + assert row["team_name"] == "finance_ops" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_usage_ignores_spoofed_execution_identity() -> None: + server, thread, port, _coord = _serve() + base = f"http://127.0.0.1:{port}" + try: + status, body = _request( + "POST", + f"{base}/v1/embeddings", + { + "model": "mock-a", + "input": "semantic search chunk for invoices", + "attribution": { + "model_name": "text-embedding-3-large", + "provider": "openai", + "company": "acme_buyer", + }, + }, + ) + assert status == 200, body + + status, records = _request("GET", f"{base}/api/v1/llm_usage_records") + assert status == 200, records + assert records["total_count"] >= 1 + # Find the embeddings channel row if mixed, else first. + rows = records["items"] + row = next( + (r for r in rows if r.get("request_channel") in {"sync", "batch"} and r.get("model_name") == "mock-a"), + rows[0], + ) + assert row["model_name"] == "mock-a" + assert "text-embedding-3-large" not in json.dumps(row) + assert row["company_name"] == "acme_buyer" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_public_cost_ledger_ignores_spoofed_execution_identity() -> None: + """Direct ledger callers cannot overwrite provider/model execution evidence.""" + + config = InMemoryConfigStore() + ledger = CostLedger(PriceBook(config)) + ledger.record_usage( + provider="actual_provider", + model="actual_model", + prompt_tokens=10, + completion_tokens=2, + attribution={ + "account": "buyer_account", + "model_name": "spoofed_model", + "provider": "spoofed_provider", + "upstream_api": "spoofed_provider", + }, + ) + rows = ledger.store.query(None, None) + assert len(rows) == 1 + assert rows[0]["model_name"] == "actual_model" + assert rows[0]["provider_name"] == "actual_provider" + assert rows[0]["upstream_api"] == "actual_provider" + assert rows[0]["account_name"] == "buyer_account" + + +if __name__ == "__main__": + test_public_cost_ledger_ignores_spoofed_execution_identity() + test_http_chat_usage_rollups_ignore_spoofed_model_name() + test_http_embeddings_usage_ignores_spoofed_execution_identity() + print("ok") diff --git a/tests/test_message_audio_function_call_http_honesty.py b/tests/test_message_audio_function_call_http_honesty.py new file mode 100644 index 000000000..5d1402297 --- /dev/null +++ b/tests/test_message_audio_function_call_http_honesty.py @@ -0,0 +1,252 @@ +"""Message audio/function_call null-empty omit and non-empty fail-closed honesty.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "message_audio_function_call_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_message_audio_null_and_empty() -> None: + server, thread, port = _server() + try: + for audio in (None, {}): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": "prior", "audio": audio}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 200, (audio, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_nonempty_message_audio() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "assistant", + "content": "prior", + "audio": {"id": "audio_1", "data": "AAAA"}, + }, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_audio" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_nonempty_message_audio_on_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hi", "audio": {"id": "audio_u"}}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_audio" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_message_function_call_null_and_empty() -> None: + server, thread, port = _server() + try: + for function_call in (None, {}): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "assistant", + "content": "prior", + "function_call": function_call, + }, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 200, (function_call, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_nonempty_message_function_call() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "assistant", + "content": "prior", + "function_call": {"name": "lookup", "arguments": "{}"}, + }, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_function_call" in blob + assert "tool_calls" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_nonempty_message_function_call_on_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": "hi", + "function_call": {"name": "lookup", "arguments": "{}"}, + }, + ], + }, + ) + assert status == 400, body + assert "invalid_message_function_call" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_message_audio_on_tools_passthrough_path() -> None: + """Passthrough (tools present) must still fail closed on message audio.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "assistant", + "content": "prior", + "audio": {"id": "audio_p"}, + }, + {"role": "user", "content": "continue"}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_audio" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_null_message_audio_on_tools_passthrough_path() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": "prior", "audio": None}, + {"role": "user", "content": "continue"}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + # mock passthrough may 200 or provider-shaped error; never 400 on null audio + assert status != 400 or "invalid_message_audio" not in json.dumps(body), body + assert status in (200, 502, 503) or "choices" in body or "error" in body, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_message_name_empty_omit_http_honesty.py b/tests/test_message_name_empty_omit_http_honesty.py new file mode 100644 index 000000000..39c0410d2 --- /dev/null +++ b/tests/test_message_name_empty_omit_http_honesty.py @@ -0,0 +1,112 @@ +"""Chat message name empty/whitespace string is omit-equivalent over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "message_name_empty_omit_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_message_name_null_empty_and_whitespace() -> None: + server, thread, port = _server() + try: + for value in (None, "", " "): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello", "name": value}], + }, + ) + assert status == 200, (value, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_message_name_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello", "name": 123}], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_name" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_message_name_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hello", "name": "n" * 65}], + }, + ) + assert status == 400, body + assert "invalid_message_name" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_message_name_null_empty_and_whitespace() + test_http_chat_rejects_message_name_non_string() + test_http_chat_rejects_message_name_too_long() + print("ok") diff --git a/tests/test_message_name_null_noop_http_honesty.py b/tests/test_message_name_null_noop_http_honesty.py new file mode 100644 index 000000000..2f2199a72 --- /dev/null +++ b/tests/test_message_name_null_noop_http_honesty.py @@ -0,0 +1,106 @@ +"""Chat message name null as omit no-op over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "message_name_null_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_message_name_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hello name null", "name": None}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_accepts_valid_message_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hello named", "name": "buyer_app"}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_message_name_as_omit() -> None: + """Empty string name is omit-equivalent (SDK optional blank).""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "hello empty name", "name": ""}, + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_message_prefix_http_honesty.py b/tests/test_message_prefix_http_honesty.py new file mode 100644 index 000000000..c8478b3dc --- /dev/null +++ b/tests/test_message_prefix_http_honesty.py @@ -0,0 +1,119 @@ +"""Chat message prefix honesty: null/false omit; true fail-closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "message_prefix_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_prefix_null_and_false() -> None: + server, thread, port = _server() + try: + for prefix in (None, False): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": "partial", "prefix": prefix}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 200, (prefix, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_prefix_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": "partial", "prefix": True}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_prefix" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_prefix_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "assistant", "content": "partial", "prefix": "yes"}, + {"role": "user", "content": "continue"}, + ], + }, + ) + assert status == 400, body + assert "invalid_message_prefix" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_prefix_null_and_false() + test_http_chat_rejects_prefix_true() + test_http_chat_rejects_prefix_non_boolean() + print("ok") diff --git a/tests/test_message_unknown_fields_http_honesty.py b/tests/test_message_unknown_fields_http_honesty.py new file mode 100644 index 000000000..59da4e220 --- /dev/null +++ b/tests/test_message_unknown_fields_http_honesty.py @@ -0,0 +1,176 @@ +"""Chat message unknown-fields and legacy function-role honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "message_unknown_fields_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_unknown_message_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": "hello", + "participant": "alice", + "custom_meta": 1, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "unknown_message_fields" in blob + assert "participant" in blob + assert "custom_meta" in blob + # Must not collapse to opaque unknown_fields at the body level. + assert body.get("error", {}).get("code") != "unknown_fields" or "message" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_known_optional_message_keys() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": "known keys only", + "name": "buyer_user", + "weight": 1, + "refusal": None, + "annotations": None, + "audio": None, + "function_call": None, + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_legacy_function_role() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + {"role": "user", "content": "call it"}, + { + "role": "function", + "name": "lookup_item", + "content": "{\"ok\":true}", + }, + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message_role" in blob + assert "tool" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_unknown_message_fields_on_tools_passthrough() -> None: + """Unknown message keys fail closed even when tools force passthrough.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "tools": [ + { + "type": "function", + "function": { + "name": "lookup_item", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + "messages": [ + { + "role": "user", + "content": "use tool", + "smuggled_field": True, + } + ], + }, + ) + assert status == 400, body + assert "unknown_message_fields" in json.dumps(body) + assert "smuggled_field" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_unknown_message_fields() + test_http_chat_accepts_known_optional_message_keys() + test_http_chat_rejects_legacy_function_role() + test_http_chat_unknown_message_fields_on_tools_passthrough() + print("ok") diff --git a/tests/test_message_weight_http_honesty.py b/tests/test_message_weight_http_honesty.py new file mode 100644 index 000000000..5b84f3b0b --- /dev/null +++ b/tests/test_message_weight_http_honesty.py @@ -0,0 +1,112 @@ +"""Chat message weight honesty: 0/1/null omit-equivalent; other values fail closed.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "message_weight_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_weight_null_zero_one() -> None: + server, thread, port = _server() + try: + for weight in (None, 0, 1, 0.0, 1.0): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": f"w={weight!r}", "weight": weight}], + }, + ) + assert status == 200, (weight, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_weight_out_of_range() -> None: + server, thread, port = _server() + try: + for weight in (2, 0.5, -1): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad weight", "weight": weight}], + }, + ) + assert status == 400, (weight, body) + assert "invalid_message_weight" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_weight_non_number() -> None: + server, thread, port = _server() + try: + for weight in ("1", True, []): + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad type", "weight": weight}], + }, + ) + assert status == 400, (weight, body) + assert "invalid_message_weight" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_weight_null_zero_one() + test_http_chat_rejects_weight_out_of_range() + test_http_chat_rejects_weight_non_number() + print("ok") diff --git a/tests/test_multimodal_content_parts_shape_http_honesty.py b/tests/test_multimodal_content_parts_shape_http_honesty.py new file mode 100644 index 000000000..edc3f5ab0 --- /dev/null +++ b/tests/test_multimodal_content_parts_shape_http_honesty.py @@ -0,0 +1,256 @@ +"""Multimodal content-part shape honesty (empty text/url, string image_url, detail).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "multimodal_content_parts_shape_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _error_blob(body: dict) -> str: + return json.dumps(body) + + +def test_http_chat_rejects_empty_text_content_part() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": ""}], + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in _error_blob(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_whitespace_text_content_part() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": " "}], + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in _error_blob(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_image_url() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": ""}, + } + ], + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in _error_blob(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_bare_string_image_url() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "caption this"}, + { + "type": "image_url", + "image_url": "https://example.com/photo.png", + }, + ], + } + ], + }, + ) + assert status == 200, body + assert body.get("object") == "chat.completion" or "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_image_url_detail_high() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "zoom"}, + { + "type": "image_url", + "image_url": { + "url": "https://example.com/photo.png", + "detail": "high", + }, + }, + ], + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_invalid_image_url_detail() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/photo.png", + "detail": "ultra", + }, + } + ], + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in _error_blob(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_omits_null_image_url_detail() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "see"}, + { + "type": "image_url", + "image_url": { + "url": "https://example.com/photo.png", + "detail": None, + }, + }, + ], + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_rejects_empty_text_content_part() + test_http_chat_rejects_whitespace_text_content_part() + test_http_chat_rejects_empty_image_url() + test_http_chat_accepts_bare_string_image_url() + test_http_chat_accepts_image_url_detail_high() + test_http_chat_rejects_invalid_image_url_detail() + test_http_chat_omits_null_image_url_detail() + print("ok") diff --git a/tests/test_multimodal_message_content_http_honesty.py b/tests/test_multimodal_message_content_http_honesty.py new file mode 100644 index 000000000..6a038f70c --- /dev/null +++ b/tests/test_multimodal_message_content_http_honesty.py @@ -0,0 +1,146 @@ +"""OpenAI multimodal content-parts (text + image_url) honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "multimodal_message_content_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_text_and_image_url_parts() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this image"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=" + }, + }, + ], + } + ], + }, + ) + assert status == 200, body + assert body.get("object") == "chat.completion" or "choices" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_unknown_content_part_type() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": [{"type": "input_audio", "input_audio": {}}], + } + ], + }, + ) + assert status == 400, body + assert "invalid_message_content" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_multipart_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": []}], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_message" in blob or "invalid_message_content" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_accepts_string_content() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "plain text"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_text_and_image_url_parts() + test_http_chat_rejects_unknown_content_part_type() + test_http_chat_rejects_empty_multipart_array() + test_http_chat_still_accepts_string_content() + print("ok") diff --git a/tests/test_openai_models_listing_http.py b/tests/test_openai_models_listing_http.py new file mode 100644 index 000000000..07f1cb3f7 --- /dev/null +++ b/tests/test_openai_models_listing_http.py @@ -0,0 +1,134 @@ +"""OpenAI GET /v1/models discovery honesty over real HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "openai_models_listing_http_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ + ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing")), + ModelAgent("coding_agent", "mock-coder", tags=("coding",)), + ModelAgent("duplicate_agent", "mock-generalist", tags=("reasoning",)), + ModelAgent("disabled_agent", "mock-disabled", tags=("writing",), disabled=True), + ] + ) + + +def _get(port: int, path: str, token: str | None = None) -> tuple[int, dict]: + headers = {"connection": "close"} + if token: + headers["authorization"] = f"Bearer {token}" + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + headers=headers, + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(orchestrator: TaskOrchestrator | None = None): + orch = orchestrator or build() + server = build_server(orch, port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_models_list_requires_bearer() -> None: + server, thread, port = _server() + try: + status, body = _get(port, "/v1/models") + assert status == 401, body + assert "unauthorized" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_models_list_unique_enabled_pool_models() -> None: + server, thread, port = _server() + try: + status, body = _get(port, "/v1/models", token=_TEST_AUTH_TOKEN) + assert status == 200, body + assert body.get("object") == "list" + ids = [item["id"] for item in body["data"]] + assert ids[0] == "contextual-orchestrator" + assert "mock-generalist" in ids + assert "mock-coder" in ids + assert ids.count("mock-generalist") == 1 + assert "mock-disabled" not in ids + for item in body["data"]: + assert item["object"] == "model" + assert "created" in item + assert "owned_by" in item + # Secret-free: no base URLs or credential hints. + blob = json.dumps(body) + assert "mock://" not in blob + assert "OPENAI_API_KEY" not in blob + assert "credential" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_models_get_by_id() -> None: + server, thread, port = _server() + try: + status, body = _get(port, "/v1/models/mock-coder", token=_TEST_AUTH_TOKEN) + assert status == 200, body + assert body.get("id") == "mock-coder" + assert body.get("object") == "model" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_models_get_missing_is_404() -> None: + server, thread, port = _server() + try: + status, body = _get(port, "/v1/models/not-a-model", token=_TEST_AUTH_TOKEN) + assert status == 404, body + assert "model_not_found" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_models_domain_helper_matches_http() -> None: + orch = build() + domain = orch.list_openai_models() + server, thread, port = _server(orch) + try: + status, body = _get(port, "/v1/models", token=_TEST_AUTH_TOKEN) + assert status == 200, body + assert body == domain + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_models_list_requires_bearer() + test_http_models_list_unique_enabled_pool_models() + test_http_models_get_by_id() + test_http_models_get_missing_is_404() + test_http_models_domain_helper_matches_http() + print("ok") diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index d50342289..2379c22da 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -35,6 +35,7 @@ def _build() -> TaskOrchestrator: def test_proxy_completion_forwards_response_format_and_returns_full_shape() -> None: orch = _build() body = { + "model": "mock-planner", "messages": [{"role": "user", "content": "extract JSON"}], "response_format": {"type": "json_schema", "json_schema": {"name": "x", "schema": {}}}, "temperature": 0.1, @@ -56,7 +57,7 @@ def test_proxy_completion_forwards_tools() -> None: orch = _build() tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}] result = orch.proxy_completion( - {"messages": [{"role": "user", "content": "call a tool"}], "tools": tools} + {"model": "mock-planner", "messages": [{"role": "user", "content": "call a tool"}], "tools": tools} ) assert result["echo"]["tools"] == tools @@ -102,6 +103,7 @@ def test_http_chat_completions_accepts_response_format_and_passes_through() -> N status, body = _post( url, { + "model": "mock-planner", "messages": [{"role": "user", "content": "give me JSON"}], "response_format": {"type": "json_object"}, }, @@ -118,7 +120,7 @@ def test_http_responses_endpoint_passes_through() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/responses" try: - status, body = _post(url, {"input": "hello", "tools": []}, token) + status, body = _post(url, {"model": "mock-planner", "input": "hello"}, token) finally: server.shutdown() assert status == 200 @@ -129,7 +131,7 @@ def test_http_plain_prompt_still_uses_orchestration_path() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" try: - status, body = _post(url, {"messages": [{"role": "user", "content": "hi"}]}, token) + status, body = _post(url, {"model": "mock-planner", "messages": [{"role": "user", "content": "hi"}]}, token) finally: server.shutdown() assert status == 200 diff --git a/tests/test_openai_sdk_control_fields_reject_http_honesty.py b/tests/test_openai_sdk_control_fields_reject_http_honesty.py new file mode 100644 index 000000000..eecf7290c --- /dev/null +++ b/tests/test_openai_sdk_control_fields_reject_http_honesty.py @@ -0,0 +1,170 @@ +"""OpenAI SDK control fields (prompt_cache_key/safety_identifier/verbosity) reject honesty.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "openai_sdk_control_fields_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_prompt_cache_key() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "cache key"}], + "prompt_cache_key": "tenant-cache-1", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_prompt_cache_key" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_safety_identifier() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "safety id"}], + "safety_identifier": "safety-abc", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_safety_identifier" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_verbosity() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "verbosity"}], + "verbosity": "high", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_verbosity" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_prompt_cache_key() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses cache key", + "prompt_cache_key": "resp-cache-1", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_prompt_cache_key" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_safety_identifier() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "completions safety", + "safety_identifier": "comp-safety", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_safety_identifier" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_baseline_chat_without_sdk_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "baseline"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_openai_user_field_http_honesty.py b/tests/test_openai_user_field_http_honesty.py new file mode 100644 index 000000000..91f8c9fdf --- /dev/null +++ b/tests/test_openai_user_field_http_honesty.py @@ -0,0 +1,173 @@ +"""OpenAI user field honesty on Completions and chat: empty fail-closed; null omit.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "openai_user_field_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "hi", "user": "buyer_account_9"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "hi", "user": ""}, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_null_user_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "hi", "user": None}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi"}], + "user": "buyer_account_9", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "hi"}], + "user": " ", + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "mock-planner", + "input": "semantic unit for buyer search", + "user": "buyer_account_9", + }, + ) + assert status == 200, body + assert body.get("object") == "list" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "x", "user": ""}, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_completions_accepts_user() + test_http_completions_rejects_empty_user() + test_http_completions_accepts_null_user_as_omit() + test_http_chat_accepts_user() + test_http_chat_rejects_empty_user() + test_http_embeddings_accepts_user() + test_http_embeddings_rejects_empty_user() + print("ok") diff --git a/tests/test_prediction_modalities_model_strip_http_honesty.py b/tests/test_prediction_modalities_model_strip_http_honesty.py new file mode 100644 index 000000000..051eac00e --- /dev/null +++ b/tests/test_prediction_modalities_model_strip_http_honesty.py @@ -0,0 +1,173 @@ +"""Empty prediction, padded modalities text, and stripped model name honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "prediction_modalities_model_strip_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_empty_prediction_string() -> None: + server, thread, port = _server() + try: + for pred in ("", " "): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "pred empty"}], + "prediction": pred, + }, + ) + assert status == 200, (pred, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_prediction_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "pred empty", "prediction": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_padded_modalities_text() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "mod pad"}], + "modalities": [" text "], + }, + ) + assert status == 200, body + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "mod empty str"}], + "modalities": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_padded_model_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": " mock-planner ", + "messages": [{"role": "user", "content": "model pad"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_padded_model_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": " mock-planner ", "input": "embed pad model"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_prediction() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "pred content"}], + "prediction": {"type": "content", "content": "x"}, + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_empty_prediction_string() + test_http_responses_accepts_empty_prediction_string() + test_http_chat_accepts_padded_modalities_text() + test_http_chat_accepts_padded_model_name() + test_http_embeddings_accepts_padded_model_name() + test_http_chat_still_rejects_nonempty_prediction() + print("ok") diff --git a/tests/test_prompt_cache_retention_reject_http_honesty.py b/tests/test_prompt_cache_retention_reject_http_honesty.py new file mode 100644 index 000000000..929fab7c5 --- /dev/null +++ b/tests/test_prompt_cache_retention_reject_http_honesty.py @@ -0,0 +1,128 @@ +"""OpenAI prompt_cache_retention reject honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "prompt_cache_retention_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_rejects_prompt_cache_retention() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "cache retention"}], + "prompt_cache_retention": "in_memory", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_prompt_cache_retention" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_prompt_cache_retention() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses cache retention", + "prompt_cache_retention": "24h", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_prompt_cache_retention" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_rejects_prompt_cache_retention() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "completions cache retention", + "prompt_cache_retention": "in_memory", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_prompt_cache_retention" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_baseline_without_prompt_cache_retention() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "baseline"}], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_reasoning_effort_none_store_stream_empty_noop_http_honesty.py b/tests/test_reasoning_effort_none_store_stream_empty_noop_http_honesty.py new file mode 100644 index 000000000..1806f2685 --- /dev/null +++ b/tests/test_reasoning_effort_none_store_stream_empty_noop_http_honesty.py @@ -0,0 +1,132 @@ +"""reasoning_effort none and empty store/stream/background omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "reasoning_effort_none_store_stream_empty_noop_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_reasoning_effort_none() -> None: + server, thread, port = _server() + try: + for effort in ("none", " none ", "\tnone\n"): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": f"effort {effort!r}"}], + "reasoning_effort": effort, + }, + ) + assert status == 200, (effort, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_reasoning_effort_none() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "effort none", "reasoning_effort": "none"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_store_stream_background_strings() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty bool strings"}], + "store": "", + "stream": " ", + "background": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_reasoning_effort_low() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "effort low"}], + "reasoning_effort": "low", + }, + ) + assert status == 400, body + assert "invalid_reasoning_effort" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_reasoning_effort_none() + test_http_completions_accepts_reasoning_effort_none() + test_http_chat_accepts_empty_store_stream_background_strings() + test_http_chat_still_rejects_reasoning_effort_low() + print("ok") diff --git a/tests/test_responses_attribution_routing_http_honesty.py b/tests/test_responses_attribution_routing_http_honesty.py new file mode 100644 index 000000000..2e167260a --- /dev/null +++ b/tests/test_responses_attribution_routing_http_honesty.py @@ -0,0 +1,152 @@ +"""Responses attribution and routing shape honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_attribution_routing_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_attribution() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "attr ok", + "attribution": {"account": "acct-1", "service": "responses_api"}, + }, + ) + assert status == 200, body + assert "unknown_fields" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_sync_routing_priority() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "routing sync", + "routing": {"channel": "sync", "priority": "interactive"}, + }, + ) + assert status == 200, body + assert "unknown_fields" not in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_routing_batch_channel() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "routing batch", + "routing": {"channel": "batch"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_routing" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_routing_latency_tolerant_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "latency tolerant", + "routing": {"latency_tolerant": True}, + }, + ) + assert status == 400, body + assert "invalid_routing" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_attribution_unknown_dimension() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bad attr", + "attribution": {"region": "us-east"}, + }, + ) + assert status == 400, body + assert "invalid_attribution" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_baseline_without_attribution_routing() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "baseline"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_responses_conversation_controls_http_honesty.py b/tests/test_responses_conversation_controls_http_honesty.py new file mode 100644 index 000000000..9c1cfb55f --- /dev/null +++ b/tests/test_responses_conversation_controls_http_honesty.py @@ -0,0 +1,162 @@ +"""Responses conversation-control fields honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_conversation_controls_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_baseline_without_conversation_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "hello baseline"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_previous_response_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello prev", + "previous_response_id": "resp_abc123", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_previous_response_id" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_conversation() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello conversation", + "conversation": "conv_abc", + }, + ) + assert status == 400, body + assert "invalid_conversation" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_truncation() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello truncation", + "truncation": "auto", + }, + ) + assert status == 400, body + assert "invalid_truncation" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_include() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello include", + "include": ["file_search_call.results"], + }, + ) + assert status == 400, body + assert "invalid_include" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_text_control() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello text control", + "text": {"format": {"type": "text"}}, + }, + ) + assert status == 400, body + assert "invalid_text" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_baseline_without_conversation_controls() + test_http_responses_rejects_previous_response_id() + test_http_responses_rejects_conversation() + test_http_responses_rejects_truncation() + test_http_responses_rejects_include() + test_http_responses_rejects_text_control() + print("ok") diff --git a/tests/test_responses_instructions_reasoning_http_honesty.py b/tests/test_responses_instructions_reasoning_http_honesty.py new file mode 100644 index 000000000..648aed30c --- /dev/null +++ b/tests/test_responses_instructions_reasoning_http_honesty.py @@ -0,0 +1,170 @@ +"""Responses API instructions and reasoning honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_instructions_reasoning_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_nonempty_instructions() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": "Be concise and factual.", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_blank_instructions() -> None: + """Empty instructions must not look like a configured system prompt.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": " ", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_instructions" in blob + assert "non-empty" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_instructions_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": ["Be concise"], + }, + ) + assert status == 400, body + assert "invalid_instructions" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_instructions_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize the ledger", + "instructions": "x" * 32_001, + }, + ) + assert status == 400, body + assert "invalid_instructions" in json.dumps(body) + assert "32000" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_reasoning_object() -> None: + """Buyers must not believe o-series reasoning controls were applied on passthrough.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "think carefully", + "reasoning": {"effort": "high"}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_reasoning" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_instructions_omitted() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello responses", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_nonempty_instructions() + test_http_responses_rejects_blank_instructions() + test_http_responses_rejects_instructions_non_string() + test_http_responses_rejects_instructions_too_long() + test_http_responses_rejects_reasoning_object() + test_http_responses_accepts_instructions_omitted() + print("ok") diff --git a/tests/test_responses_logit_bias_logprobs_http_honesty.py b/tests/test_responses_logit_bias_logprobs_http_honesty.py new file mode 100644 index 000000000..b3de1ddc6 --- /dev/null +++ b/tests/test_responses_logit_bias_logprobs_http_honesty.py @@ -0,0 +1,176 @@ +"""Responses logit_bias and logprobs shape honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_logit_bias_logprobs_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_empty_and_valid_logit_bias() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "bias empty", "logit_bias": {}}, + ) + assert status == 200, body + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bias map", + "logit_bias": {"50256": -100, "220": 50}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_digit_logit_bias_key() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bias bad key", + "logit_bias": {"not-a-token": 1}, + }, + ) + assert status == 400, body + assert "invalid_logit_bias" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_out_of_range_logit_bias_value() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bias bad value", + "logit_bias": {"100": 101}, + }, + ) + assert status == 400, body + assert "invalid_logit_bias" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_logprobs_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "lp false", "logprobs": False}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_logprobs_true_with_top_logprobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "lp true", + "logprobs": True, + "top_logprobs": 5, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_top_logprobs_without_logprobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "tlp alone", "top_logprobs": 5}, + ) + assert status == 400, body + assert "invalid_top_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_boolean_logprobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "lp int", "logprobs": 5}, + ) + assert status == 400, body + assert "invalid_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_empty_and_valid_logit_bias() + test_http_responses_rejects_non_digit_logit_bias_key() + test_http_responses_rejects_out_of_range_logit_bias_value() + test_http_responses_accepts_logprobs_false() + test_http_responses_accepts_logprobs_true_with_top_logprobs() + test_http_responses_rejects_top_logprobs_without_logprobs() + test_http_responses_rejects_non_boolean_logprobs() + print("ok") diff --git a/tests/test_responses_max_output_tokens_http_honesty.py b/tests/test_responses_max_output_tokens_http_honesty.py new file mode 100644 index 000000000..5bf63db61 --- /dev/null +++ b/tests/test_responses_max_output_tokens_http_honesty.py @@ -0,0 +1,154 @@ +"""Responses max_output_tokens (OpenAI-native budget) honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_max_output_tokens_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_valid_max_output_tokens() -> None: + """Official Responses clients send max_output_tokens — must not be unknown_fields.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello max_output_tokens", + "max_output_tokens": 256, + }, + ) + assert status == 200, body + blob = json.dumps(body) + assert "unknown_fields" not in blob + assert "invalid_max_output_tokens" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_omit_max_output_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello omit budget"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_zero_max_output_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "zero budget", + "max_output_tokens": 0, + }, + ) + assert status == 400, body + assert "invalid_max_output_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_integer_max_output_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "float budget", + "max_output_tokens": 1.5, + }, + ) + assert status == 400, body + assert "invalid_max_output_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_boolean_max_output_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bool budget", + "max_output_tokens": True, + }, + ) + assert status == 400, body + assert "invalid_max_output_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_oversize_max_output_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "huge budget", + "max_output_tokens": 2_000_000, + }, + ) + assert status == 400, body + assert "invalid_max_output_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_responses_max_tokens_http_honesty.py b/tests/test_responses_max_tokens_http_honesty.py new file mode 100644 index 000000000..2dc16f85b --- /dev/null +++ b/tests/test_responses_max_tokens_http_honesty.py @@ -0,0 +1,152 @@ +"""Responses max_tokens / max_completion_tokens honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_max_tokens_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_valid_max_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello max_tokens", "max_tokens": 256}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_valid_max_completion_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello mct", + "max_completion_tokens": 512, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_zero_max_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hi", "max_tokens": 0}, + ) + assert status == 400, body + assert "invalid_max_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_integer_max_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hi", "max_tokens": 1.5}, + ) + assert status == 400, body + assert "invalid_max_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_oversize_max_completion_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hi", + "max_completion_tokens": 2_000_000, + }, + ) + assert status == 400, body + assert "invalid_max_completion_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_boolean_max_completion_tokens() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hi", + "max_completion_tokens": True, + }, + ) + assert status == 400, body + assert "invalid_max_completion_tokens" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_valid_max_tokens() + test_http_responses_accepts_valid_max_completion_tokens() + test_http_responses_rejects_zero_max_tokens() + test_http_responses_rejects_non_integer_max_tokens() + test_http_responses_rejects_oversize_max_completion_tokens() + test_http_responses_rejects_boolean_max_completion_tokens() + print("ok") diff --git a/tests/test_responses_max_tool_calls_reject_http_honesty.py b/tests/test_responses_max_tool_calls_reject_http_honesty.py new file mode 100644 index 000000000..bda67d065 --- /dev/null +++ b/tests/test_responses_max_tool_calls_reject_http_honesty.py @@ -0,0 +1,99 @@ +"""Responses max_tool_calls reject honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_max_tool_calls_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_without_max_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "hello no tool budget"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_max_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello max_tool_calls", + "max_tool_calls": 3, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_max_tool_calls" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_max_tool_calls_one() -> None: + """Even max_tool_calls=1 is unsupported — no tool loop on passthrough.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello max_tool_calls 1", + "max_tool_calls": 1, + }, + ) + assert status == 400, body + assert "invalid_max_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_responses_metadata_http_honesty.py b/tests/test_responses_metadata_http_honesty.py new file mode 100644 index 000000000..c15eef3f5 --- /dev/null +++ b/tests/test_responses_metadata_http_honesty.py @@ -0,0 +1,133 @@ +"""Responses API metadata honesty over HTTP (OpenAI string-map shape).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_metadata_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_string_metadata_map() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": {"tenant_id": "buyer-9", "channel": "api"}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_object_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": "not-an-object", + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_string_metadata_value() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": {"count": 3}, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_too_many_metadata_entries() -> None: + server, thread, port = _server() + try: + meta = {f"k{i}": f"v{i}" for i in range(17)} + status, body = _post( + port, + { + "model": "mock-planner", + "input": "summarize ledger", + "metadata": meta, + }, + ) + assert status == 400, body + assert "invalid_metadata" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_omits_metadata_ok() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "summarize ledger"}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_responses_modalities_prediction_http_honesty.py b/tests/test_responses_modalities_prediction_http_honesty.py new file mode 100644 index 000000000..8121efd51 --- /dev/null +++ b/tests/test_responses_modalities_prediction_http_honesty.py @@ -0,0 +1,129 @@ +"""Responses modalities / prediction / reasoning_effort honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_modalities_prediction_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_text_modalities() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "text only", + "modalities": ["text"], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_audio_modalities() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "audio", + "modalities": ["text", "audio"], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_modalities" in blob + assert "text" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_prediction() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "pred", + "prediction": {"type": "content", "content": "x"}, + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_reasoning_effort() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "effort", + "reasoning_effort": "high", + }, + ) + assert status == 400, body + assert "invalid_reasoning_effort" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_text_modalities() + test_http_responses_rejects_audio_modalities() + test_http_responses_rejects_prediction() + test_http_responses_rejects_reasoning_effort() + print("ok") diff --git a/tests/test_responses_model_required_http_honesty.py b/tests/test_responses_model_required_http_honesty.py new file mode 100644 index 000000000..29b4ef541 --- /dev/null +++ b/tests/test_responses_model_required_http_honesty.py @@ -0,0 +1,118 @@ +"""Responses API model field required honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_model_required_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_rejects_missing_model() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"input": "hello"}) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_model" in blob + assert "required" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_empty_model() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": " ", "input": "hello"}) + assert status == 400, body + assert "invalid_model" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_string_model() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": 12, "input": "hello"}) + assert status == 400, body + assert "invalid_model" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_overlong_model() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "m" * 257, "input": "hello"}) + assert status == 400, body + assert "invalid_model" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_pool_model() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "input": "hello"}) + assert status == 200, body + # OpenAI Responses shape or chat-compatible framing + assert "output" in body or "choices" in body or body.get("object") in { + "response", + "chat.completion", + } or "id" in body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_rejects_missing_model() + test_http_responses_rejects_empty_model() + test_http_responses_rejects_non_string_model() + test_http_responses_rejects_overlong_model() + test_http_responses_accepts_pool_model() + print("ok") diff --git a/tests/test_responses_n_http_honesty.py b/tests/test_responses_n_http_honesty.py new file mode 100644 index 000000000..5c062e3e5 --- /dev/null +++ b/tests/test_responses_n_http_honesty.py @@ -0,0 +1,106 @@ +"""Responses n multi-choice honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_n_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_omitted_n() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "input": "hello n omit"}) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_n_equals_one() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "hello n one", "n": 1} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_n_greater_than_one() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "hello n multi", "n": 2} + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_n" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_integer_n() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "hello n bad", "n": 1.5} + ) + assert status == 400, body + assert "invalid_n" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_omitted_n() + test_http_responses_accepts_n_equals_one() + test_http_responses_rejects_n_greater_than_one() + test_http_responses_rejects_non_integer_n() + print("ok") diff --git a/tests/test_responses_parallel_tool_calls_http_honesty.py b/tests/test_responses_parallel_tool_calls_http_honesty.py new file mode 100644 index 000000000..664bb77e5 --- /dev/null +++ b/tests/test_responses_parallel_tool_calls_http_honesty.py @@ -0,0 +1,165 @@ +"""Responses parallel_tool_calls honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_parallel_tool_calls_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_omitted_parallel_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "input": "hello ptc omit"}) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_false_parallel_tool_calls_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello ptc false", + "parallel_tool_calls": False, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_true_parallel_tool_calls_with_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello ptc true with tools", + "parallel_tool_calls": True, + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_true_parallel_tool_calls_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello ptc true no tools", + "parallel_tool_calls": True, + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_boolean_parallel_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello ptc bad", + "parallel_tool_calls": "yes", + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_integer_parallel_tool_calls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello ptc int", + "parallel_tool_calls": 1, + }, + ) + assert status == 400, body + assert "invalid_parallel_tool_calls" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_omitted_parallel_tool_calls() + test_http_responses_accepts_false_parallel_tool_calls_without_tools() + test_http_responses_accepts_true_parallel_tool_calls_with_tools() + test_http_responses_rejects_true_parallel_tool_calls_without_tools() + test_http_responses_rejects_non_boolean_parallel_tool_calls() + test_http_responses_rejects_integer_parallel_tool_calls() + print("ok") diff --git a/tests/test_responses_penalties_http_honesty.py b/tests/test_responses_penalties_http_honesty.py new file mode 100644 index 000000000..a77f59870 --- /dev/null +++ b/tests/test_responses_penalties_http_honesty.py @@ -0,0 +1,128 @@ +"""Responses presence/frequency penalty honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_penalties_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_valid_penalties() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello penalties", + "presence_penalty": 0.5, + "frequency_penalty": -0.5, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_out_of_range_presence_penalty() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hi", + "presence_penalty": 3.0, + }, + ) + assert status == 400, body + assert "invalid_presence_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_out_of_range_frequency_penalty() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hi", + "frequency_penalty": -3.0, + }, + ) + assert status == 400, body + assert "invalid_frequency_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_boolean_presence_penalty() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hi", + "presence_penalty": True, + }, + ) + assert status == 400, body + assert "invalid_presence_penalty" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_valid_penalties() + test_http_responses_rejects_out_of_range_presence_penalty() + test_http_responses_rejects_out_of_range_frequency_penalty() + test_http_responses_rejects_boolean_presence_penalty() + print("ok") diff --git a/tests/test_responses_response_format_http_honesty.py b/tests/test_responses_response_format_http_honesty.py new file mode 100644 index 000000000..fc46a2f5b --- /dev/null +++ b/tests/test_responses_response_format_http_honesty.py @@ -0,0 +1,155 @@ +"""Responses response_format shape honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_response_format_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_text_and_json_object_format() -> None: + server, thread, port = _server() + try: + for fmt in ({"type": "text"}, {"type": "json_object"}): + status, body = _post( + port, + { + "model": "mock-planner", + "input": f"format {fmt['type']}", + "response_format": fmt, + }, + ) + assert status == 200, (fmt, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_json_schema_format() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "schema format", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "item_schema", + "schema": {"type": "object", "properties": {}}, + }, + }, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_unknown_response_format_type() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bad type", + "response_format": {"type": "xml"}, + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_json_object_with_extra_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "extra", + "response_format": {"type": "json_object", "strict": True}, + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_json_schema_without_schema_body() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "missing schema", + "response_format": { + "type": "json_schema", + "json_schema": {"name": "item_schema"}, + }, + }, + ) + assert status == 400, body + assert "invalid_response_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_text_and_json_object_format() + test_http_responses_accepts_json_schema_format() + test_http_responses_rejects_unknown_response_format_type() + test_http_responses_rejects_json_object_with_extra_fields() + test_http_responses_rejects_json_schema_without_schema_body() + print("ok") diff --git a/tests/test_responses_seed_stop_http_honesty.py b/tests/test_responses_seed_stop_http_honesty.py new file mode 100644 index 000000000..8af00eeea --- /dev/null +++ b/tests/test_responses_seed_stop_http_honesty.py @@ -0,0 +1,168 @@ +"""Responses seed and stop shape honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_seed_stop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_valid_seed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello seed", "seed": 42}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_integer_seed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello seed", "seed": 1.5}, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_boolean_seed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello seed", "seed": True}, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_stop_string_and_array() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello stop str", "stop": "END"}, + ) + assert status == 200, body + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello stop arr", + "stop": ["END", "STOP"], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_stop_string_as_omit() -> None: + """Empty stop string is an omit-equivalent SDK optional default.""" + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello stop", "stop": ""}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_stop_array_too_long() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello stop", + "stop": ["a", "b", "c", "d", "e"], + }, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_string_stop_item() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello stop", "stop": [12]}, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_valid_seed() + test_http_responses_rejects_non_integer_seed() + test_http_responses_rejects_boolean_seed() + test_http_responses_accepts_stop_string_and_array() + test_http_responses_accepts_empty_stop_string_as_omit() + test_http_responses_rejects_stop_array_too_long() + test_http_responses_rejects_non_string_stop_item() + print("ok") diff --git a/tests/test_responses_service_tier_http_honesty.py b/tests/test_responses_service_tier_http_honesty.py new file mode 100644 index 000000000..51142d6f5 --- /dev/null +++ b/tests/test_responses_service_tier_http_honesty.py @@ -0,0 +1,141 @@ +"""Responses API service_tier honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_service_tier_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_omitted_service_tier() -> None: + server, thread, port = _server() + try: + status, body = _post(port, {"model": "mock-planner", "input": "hello tier omit"}) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_auto_and_default_service_tier() -> None: + server, thread, port = _server() + try: + for tier in ("auto", "default"): + status, body = _post( + port, + { + "model": "mock-planner", + "input": f"hello tier {tier}", + "service_tier": tier, + }, + ) + assert status == 200, (tier, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_flex_service_tier() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello tier flex", + "service_tier": "flex", + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_service_tier" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_priority_service_tier() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello tier priority", + "service_tier": "priority", + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_string_service_tier() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello tier bad", + "service_tier": 1, + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_omitted_service_tier() + test_http_responses_accepts_auto_and_default_service_tier() + test_http_responses_rejects_flex_service_tier() + test_http_responses_rejects_priority_service_tier() + test_http_responses_rejects_non_string_service_tier() + print("ok") diff --git a/tests/test_responses_store_http_honesty.py b/tests/test_responses_store_http_honesty.py new file mode 100644 index 000000000..682200871 --- /dev/null +++ b/tests/test_responses_store_http_honesty.py @@ -0,0 +1,124 @@ +"""Responses API store field honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_store_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_omitted_store() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "hello responses store omit"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_store_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello responses store false", + "store": False, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_store_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello responses store true", + "store": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_store" in blob + assert "not supported" in blob + assert "responses" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_boolean_store() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello responses store bad", + "store": "yes", + }, + ) + assert status == 400, body + assert "invalid_store" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_omitted_store() + test_http_responses_accepts_store_false() + test_http_responses_rejects_store_true() + test_http_responses_rejects_non_boolean_store() + print("ok") diff --git a/tests/test_responses_stream_options_http_honesty.py b/tests/test_responses_stream_options_http_honesty.py new file mode 100644 index 000000000..41c9dc8e0 --- /dev/null +++ b/tests/test_responses_stream_options_http_honesty.py @@ -0,0 +1,125 @@ +"""Responses stream_options honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_stream_options_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_without_stream_options() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, {"model": "mock-planner", "input": "hello no stream_options"} + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_all_false_stream_options_as_omit() -> None: + """All-false flags match chat/Completions omit-equivalent SDK defaults.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello stream_options false", + "stream_options": {"include_usage": False}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_stream_options_true_flag() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello stream_options true", + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stream_options" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_stream_options_with_stream_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello stream false opts", + "stream": False, + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_without_stream_options() + test_http_responses_accepts_all_false_stream_options_as_omit() + test_http_responses_rejects_stream_options_true_flag() + test_http_responses_rejects_stream_options_with_stream_false() + print("ok") diff --git a/tests/test_responses_stream_reject_http_honesty.py b/tests/test_responses_stream_reject_http_honesty.py new file mode 100644 index 000000000..be5147f86 --- /dev/null +++ b/tests/test_responses_stream_reject_http_honesty.py @@ -0,0 +1,120 @@ +"""Responses stream=true reject honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_stream_reject_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_stream_false() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello no stream", + "stream": False, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_stream_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello omit stream", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_stream_true() -> None: + """Responses passthrough has no SSE plane — stream=true fails closed.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello stream true", + "stream": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stream" in blob + assert "not supported" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_stream_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello stream string", + "stream": "yes", + }, + ) + assert status == 400, body + assert "invalid_stream" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_responses_temperature_top_p_http_honesty.py b/tests/test_responses_temperature_top_p_http_honesty.py new file mode 100644 index 000000000..21cc35e50 --- /dev/null +++ b/tests/test_responses_temperature_top_p_http_honesty.py @@ -0,0 +1,131 @@ +"""Responses temperature/top_p shape honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_temperature_top_p_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_valid_temperature_and_top_p() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello sampling", + "temperature": 0.7, + "top_p": 0.9, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_out_of_range_temperature() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hi", "temperature": 3.0}, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_numeric_temperature() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hi", "temperature": "hot"}, + ) + assert status == 400, body + assert "invalid_temperature" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_out_of_range_top_p() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hi", "top_p": 0.0}, + ) + assert status == 400, body + assert "invalid_top_p" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_boolean_top_p() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hi", "top_p": True}, + ) + assert status == 400, body + assert "invalid_top_p" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_valid_temperature_and_top_p() + test_http_responses_rejects_out_of_range_temperature() + test_http_responses_rejects_non_numeric_temperature() + test_http_responses_rejects_out_of_range_top_p() + test_http_responses_rejects_boolean_top_p() + print("ok") diff --git a/tests/test_responses_tools_shape_http_honesty.py b/tests/test_responses_tools_shape_http_honesty.py new file mode 100644 index 000000000..0b50d86db --- /dev/null +++ b/tests/test_responses_tools_shape_http_honesty.py @@ -0,0 +1,181 @@ +"""Responses tools / tool_choice shape honesty over HTTP (fail-closed).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_tools_shape_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _valid_tools() -> list[dict]: + return [ + { + "type": "function", + "function": { + "name": "lookup_item", + "description": "look up an item", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + +def test_http_responses_accepts_valid_tools_and_auto_choice() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "use tools", + "tools": _valid_tools(), + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_empty_tools_array_as_noop() -> None: + """SDKs often send tools: [] when no tools are configured — honest no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "tools empty", "tools": []}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_tool_without_function_type() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "bad tool type", + "tools": [{"type": "retrieval", "function": {"name": "x"}}], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_tool_choice_auto_without_tools_as_omit() -> None: + """tool_choice auto/none without tools is an omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "choice alone", + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_legacy_functions_surface() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "legacy functions", + "functions": [{"name": "lookup_item", "parameters": {}}], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_functions" in blob + assert "tools" in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_named_tool_choice_not_in_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "named missing", + "tools": _valid_tools(), + "tool_choice": { + "type": "function", + "function": {"name": "not_declared"}, + }, + }, + ) + assert status == 400, body + assert "invalid_tool_choice" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_valid_tools_and_auto_choice() + test_http_responses_rejects_empty_tools_array() + test_http_responses_rejects_tool_without_function_type() + test_http_responses_accepts_tool_choice_auto_without_tools_as_omit() + test_http_responses_rejects_legacy_functions_surface() + test_http_responses_rejects_named_tool_choice_not_in_tools() + print("ok") diff --git a/tests/test_responses_user_field_http_honesty.py b/tests/test_responses_user_field_http_honesty.py new file mode 100644 index 000000000..54e8c40f8 --- /dev/null +++ b/tests/test_responses_user_field_http_honesty.py @@ -0,0 +1,129 @@ +"""Responses API OpenAI user field honesty over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "responses_user_field_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/responses", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_responses_accepts_valid_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "input": "hello user field", + "user": "buyer_account_42", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello", "user": " "}, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) or "user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_null_user_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello", "user": None}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_non_string_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello", "user": 99}, + ) + assert status == 400, body + assert "user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_overlong_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + {"model": "mock-planner", "input": "hello", "user": "u" * 65}, + ) + assert status == 400, body + assert "user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_responses_accepts_valid_user() + test_http_responses_rejects_empty_user() + test_http_responses_accepts_null_user_as_omit() + test_http_responses_rejects_non_string_user() + test_http_responses_rejects_overlong_user() + print("ok") diff --git a/tests/test_sales_readiness.py b/tests/test_sales_readiness.py index d690d5573..0ede97689 100644 --- a/tests/test_sales_readiness.py +++ b/tests/test_sales_readiness.py @@ -160,7 +160,7 @@ def test_sales_readiness_endpoint_openapi_and_admin_surface() -> None: ) chat_status, _ = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "Analyze, verify, and summarize readiness."}]}, + {"model": "mock-planner", "messages": [{"role": "user", "content": "Analyze, verify, and summarize readiness."}]}, "inference_secret", ) readiness_status, readiness = get_json( diff --git a/tests/test_sdk_null_legacy_controls_noop_http_honesty.py b/tests/test_sdk_null_legacy_controls_noop_http_honesty.py new file mode 100644 index 000000000..29d5f284f --- /dev/null +++ b/tests/test_sdk_null_legacy_controls_noop_http_honesty.py @@ -0,0 +1,155 @@ +"""SDK null legacy/control fields as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "sdk_null_legacy_controls_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_completions_accepts_null_echo_suffix_best_of() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "null legacy", + "echo": None, + "suffix": None, + "best_of": None, + "logprobs": None, + "top_logprobs": None, + "prompt_cache_key": None, + "verbosity": None, + "include": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_null_conversation_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "null conversation controls", + "previous_response_id": None, + "conversation": None, + "truncation": None, + "include": None, + "text": None, + "logprobs": None, + "top_logprobs": None, + "prompt_cache_retention": None, + "safety_identifier": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_null_include_and_sdk_controls() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null include"}], + "include": None, + "prompt_cache_key": None, + "verbosity": None, + "logprobs": None, + "top_logprobs": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_echo_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "echo true", "echo": True}, + ) + assert status == 400, body + assert "invalid_echo" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_still_rejects_previous_response_id() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "prev", + "previous_response_id": "resp_123", + }, + ) + assert status == 400, body + assert "invalid_previous_response_id" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_sdk_null_object_optional_noop_http_honesty.py b/tests/test_sdk_null_object_optional_noop_http_honesty.py new file mode 100644 index 000000000..adab99ea8 --- /dev/null +++ b/tests/test_sdk_null_object_optional_noop_http_honesty.py @@ -0,0 +1,227 @@ +"""SDK null object/optional fields as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "sdk_null_object_optional_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_null_metadata_tools_tool_choice() -> None: + """Null object/optionals are omit no-ops on the multi-agent chat path.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null objects chat"}], + "metadata": None, + "tools": None, + "tool_choice": None, + "response_format": None, + "logit_bias": None, + "modalities": None, + "stream_options": None, + "reasoning_effort": None, + "prediction": None, + "background": None, + "logprobs": None, + "top_logprobs": None, + }, + ) + assert status == 200, body + # Orchestration path (null tools/response_format must not force passthrough). + assert body.get("object") == "chat.completion", body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_null_stream_options_alone() -> None: + """stream_options:null must not 400 when stream is omitted (orchestration path).""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null stream_options alone"}], + "stream_options": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_null_stream_logit_bias_metadata() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "null objects completions", + "stream": None, + "logit_bias": None, + "metadata": None, + "response_format": None, + "background": None, + "tools": None, + "tool_choice": None, + "functions": None, + "function_call": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_null_instructions_metadata_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "null objects responses", + "instructions": None, + "metadata": None, + "tools": None, + "tool_choice": None, + "logit_bias": None, + "modalities": None, + "background": None, + "stream_options": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonzero_reasoning_effort() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "effort high"}], + "reasoning_effort": "high", + }, + ) + assert status == 400, body + assert "invalid_reasoning_effort" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_prediction_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "predict obj"}], + "prediction": {"type": "content", "content": "x"}, + }, + ) + assert status == 400, body + assert "invalid_prediction" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_all_false_stream_options_without_stream() -> None: + """All-false stream_options without stream is an omit-equivalent no-op.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stream opts"}], + "stream_options": {"include_usage": False}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_true_stream_options_without_stream() -> None: + """True stream_options flags without stream=true still fail closed.""" + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stream opts true"}], + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_sdk_null_optional_noop_http_honesty.py b/tests/test_sdk_null_optional_noop_http_honesty.py new file mode 100644 index 000000000..a88643db0 --- /dev/null +++ b/tests/test_sdk_null_optional_noop_http_honesty.py @@ -0,0 +1,183 @@ +"""SDK optional null fields as omit no-ops over HTTP (chat/completions/responses).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "sdk_null_optional_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_null_service_tier_store_seed_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null optionals chat"}], + "service_tier": None, + "store": None, + "seed": None, + "stop": None, + "n": None, + "temperature": None, + "top_p": None, + "max_tokens": None, + "max_completion_tokens": None, + "frequency_penalty": None, + "presence_penalty": None, + "parallel_tool_calls": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_null_seed_stop_store_n() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "null optionals completions", + "seed": None, + "stop": None, + "store": None, + "n": None, + "service_tier": None, + "temperature": None, + "max_tokens": None, + "parallel_tool_calls": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_null_seed_stop_store_n() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "null optionals responses", + "seed": None, + "stop": None, + "store": None, + "n": None, + "service_tier": None, + "max_output_tokens": None, + "parallel_tool_calls": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonzero_seed() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "seed 1"}], + "seed": 1, + }, + ) + assert status == 400, body + assert "invalid_seed" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_store_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "store true"}], + "store": True, + }, + ) + assert status == 400, body + assert "invalid_store" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_nonnull_stop() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "stop word", + "stop": "END", + }, + ) + assert status == 400, body + assert "invalid_stop" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index 67134ea6b..d7ccc5083 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -42,7 +42,7 @@ def test_http_api_requires_bearer_token_and_hides_trace_by_default() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: unauthorized_status, unauthorized_body = post_json(f"http://127.0.0.1:{port}/v1/chat/completions", payload) @@ -72,7 +72,7 @@ def test_admin_and_inference_tokens_are_separate() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: admin_for_chat_status, _ = post_json( @@ -100,7 +100,7 @@ def test_loopback_without_configured_token_is_rejected() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: status, body = post_json(f"http://127.0.0.1:{port}/v1/chat/completions", payload) @@ -121,7 +121,7 @@ def test_http_api_validates_mode_and_request_shape() -> None: try: status, body = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "owner", "content": "hello"}], "orchestration": "unsafe"}, + {"model": "mock-generalist", "messages": [{"role": "owner", "content": "hello"}], "orchestration": "unsafe"}, token="secret_token", ) finally: @@ -141,7 +141,7 @@ def test_http_api_rejects_unknown_request_fields() -> None: try: status, body = post_json( f"http://127.0.0.1:{port}/v1/chat/completions", - {"messages": [{"role": "user", "content": "hello"}], "unexpected": True}, + {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}], "unexpected": True}, token="secret_token", ) finally: @@ -161,7 +161,7 @@ def test_rate_limit_returns_429_after_configured_budget() -> None: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] - payload = {"messages": [{"role": "user", "content": "hello"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "hello"}]} try: first_status, _ = post_json(f"http://127.0.0.1:{port}/v1/chat/completions", payload, token="secret_token") diff --git a/tests/test_service_tier_encoding_format_strip_http_honesty.py b/tests/test_service_tier_encoding_format_strip_http_honesty.py new file mode 100644 index 000000000..a94e03b9a --- /dev/null +++ b/tests/test_service_tier_encoding_format_strip_http_honesty.py @@ -0,0 +1,191 @@ +"""Whitespace-padded service_tier auto/default and encoding_format float honesty.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "service_tier_encoding_format_strip_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_padded_service_tier_auto() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tier pad"}], + "service_tier": " auto ", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_padded_service_tier_default() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tier default pad"}], + "service_tier": "\tdefault\n", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_padded_service_tier_default() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "tier completions pad", + "service_tier": " default ", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_padded_service_tier_auto() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "tier responses pad", + "service_tier": "\tauto\t", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_flex_service_tier() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tier flex"}], + "service_tier": " flex ", + }, + ) + assert status == 400, body + assert "invalid_service_tier" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_padded_encoding_format_float() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "mock-planner", + "input": "encode pad float", + "encoding_format": " float ", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_still_rejects_padded_base64() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + { + "model": "mock-planner", + "input": "encode pad base64", + "encoding_format": " base64 ", + }, + ) + assert status == 400, body + assert "invalid_encoding_format" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_padded_service_tier_auto() + test_http_chat_accepts_padded_service_tier_default() + test_http_completions_accepts_padded_service_tier_default() + test_http_responses_accepts_padded_service_tier_auto() + test_http_chat_still_rejects_flex_service_tier() + test_http_embeddings_accepts_padded_encoding_format_float() + test_http_embeddings_still_rejects_padded_base64() + print("ok") diff --git a/tests/test_stream_null_noop_http_honesty.py b/tests/test_stream_null_noop_http_honesty.py new file mode 100644 index 000000000..30dcb75fb --- /dev/null +++ b/tests/test_stream_null_noop_http_honesty.py @@ -0,0 +1,120 @@ +"""stream/stream_options null as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "stream_null_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_stream_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "stream null"}], + "stream": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_stream_null_and_false() -> None: + server, thread, port = _server() + try: + for stream in (None, False): + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "stream val", "stream": stream}, + ) + assert status == 200, (stream, body) + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "stream_options null", + "stream_options": None, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_still_rejects_stream_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "stream true", "stream": True}, + ) + assert status == 400, body + assert "invalid_stream" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_stream_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "stream null", "stream": None}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_stream_options_false_tool_choice_empty_noop_http_honesty.py b/tests/test_stream_options_false_tool_choice_empty_noop_http_honesty.py new file mode 100644 index 000000000..db62cc689 --- /dev/null +++ b/tests/test_stream_options_false_tool_choice_empty_noop_http_honesty.py @@ -0,0 +1,164 @@ +"""All-false stream_options, empty tool_choice {}, empty reasoning_effort as omit.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "stream_options_false_tool_choice_empty_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_all_false_stream_options_without_stream() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "all false so"}], + "stream": False, + "stream_options": { + "include_usage": False, + "include_obfuscation": False, + }, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_all_false_stream_options_without_stream() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "all false so", + "stream": False, + "stream_options": {"include_usage": False}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_tool_choice_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty tool_choice"}], + "tool_choice": {}, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_empty_string_reasoning_effort() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty effort"}], + "reasoning_effort": "", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_stream_options_true_without_stream() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "so true"}], + "stream": False, + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonempty_reasoning_effort() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "effort high"}], + "reasoning_effort": "high", + }, + ) + assert status == 400, body + assert "invalid_reasoning_effort" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_stream_options_null_flags_noop_http_honesty.py b/tests/test_stream_options_null_flags_noop_http_honesty.py new file mode 100644 index 000000000..fe5b99e61 --- /dev/null +++ b/tests/test_stream_options_null_flags_noop_http_honesty.py @@ -0,0 +1,173 @@ +"""stream_options null flag values are omit-equivalent no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "stream_options_null_flags_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_stream_options_null_flags_without_stream() -> None: + """SDK optional null flags must not require stream=true or fail type checks.""" + server, thread, port = _server() + try: + for opts in ( + {"include_usage": None}, + {"include_obfuscation": None}, + {"include_usage": None, "include_obfuscation": None}, + {"include_usage": None, "include_obfuscation": False}, + {"include_usage": False, "include_obfuscation": None}, + ): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "null so flags"}], + "stream": False, + "stream_options": opts, + }, + ) + assert status == 200, (opts, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_stream_options_null_flags_without_stream() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + { + "model": "mock-planner", + "prompt": "null so flags", + "stream": False, + "stream_options": { + "include_usage": None, + "include_obfuscation": None, + }, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_stream_options_null_flags() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "null so flags", + "stream_options": { + "include_usage": None, + "include_obfuscation": None, + }, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_include_usage_true() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "usage true"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_stream_options" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_non_boolean_non_null_flag() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "bad flag"}], + "stream": True, + "stream_options": {"include_usage": "yes"}, + }, + ) + assert status == 400, body + assert "invalid_stream_options" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_stream_options_null_flags_without_stream() + test_http_completions_accepts_stream_options_null_flags_without_stream() + test_http_responses_accepts_stream_options_null_flags() + test_http_chat_still_rejects_include_usage_true() + test_http_chat_rejects_non_boolean_non_null_flag() + print("ok") diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 789fb7084..f1dd81e06 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -89,7 +89,7 @@ def _serve() -> tuple[object, int, str]: def test_http_stream_true_returns_event_stream_and_reconstructs_answer() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" - payload = {"messages": [{"role": "user", "content": "stream please"}]} + payload = {"model": "mock-generalist", "messages": [{"role": "user", "content": "stream please"}]} try: # Non-streaming reference answer. _, ref_ct, ref_body = _post(url, payload, token) @@ -118,7 +118,7 @@ def test_http_stream_false_is_unchanged_json() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" try: - status, content_type, body = _post(url, {"messages": [{"role": "user", "content": "hi"}], "stream": False}, token) + status, content_type, body = _post(url, {"model": "mock-generalist", "messages": [{"role": "user", "content": "hi"}], "stream": False}, token) finally: server.shutdown() assert status == 200 @@ -132,7 +132,7 @@ def test_http_stream_non_boolean_is_rejected() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" try: - status, _, body = _post(url, {"messages": [{"role": "user", "content": "hi"}], "stream": "yes"}, token) + status, _, body = _post(url, {"model": "mock-generalist", "messages": [{"role": "user", "content": "hi"}], "stream": "yes"}, token) finally: server.shutdown() assert status == 400 diff --git a/tests/test_tool_choice_auto_without_tools_noop_http_honesty.py b/tests/test_tool_choice_auto_without_tools_noop_http_honesty.py new file mode 100644 index 000000000..c08b6ecfc --- /dev/null +++ b/tests/test_tool_choice_auto_without_tools_noop_http_honesty.py @@ -0,0 +1,140 @@ +"""tool_choice none/auto without tools as omit no-ops over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "tool_choice_auto_without_tools_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_tool_choice_auto_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "auto no tools"}], + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_tool_choice_none_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "none no tools"}], + "tool_choice": "none", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_tool_choice_auto_with_empty_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "auto empty tools"}], + "tools": [], + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_tool_choice_auto_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "auto no tools", + "tool_choice": "auto", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_tool_choice_required_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "required no tools"}], + "tool_choice": "required", + }, + ) + assert status == 400, body + assert "invalid_tool_choice" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) diff --git a/tests/test_tool_choice_strip_modalities_text_noop_http_honesty.py b/tests/test_tool_choice_strip_modalities_text_noop_http_honesty.py new file mode 100644 index 000000000..a122b4839 --- /dev/null +++ b/tests/test_tool_choice_strip_modalities_text_noop_http_honesty.py @@ -0,0 +1,169 @@ +"""Whitespace-padded tool_choice/function_call none/auto and Completions modalities text no-ops.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "tool_choice_strip_modalities_text_noop_http_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_padded_tool_choice_none_auto() -> None: + server, thread, port = _server() + try: + for tc in (" none ", "\tauto\n", " none"): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": f"tc {tc!r}"}], + "tool_choice": tc, + }, + ) + assert status == 200, (tc, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_padded_function_call_none_auto() -> None: + server, thread, port = _server() + try: + for fc in (" none ", " auto "): + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": f"fc {fc!r}"}], + "function_call": fc, + }, + ) + assert status == 200, (fc, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_padded_tool_choice_function_call() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "tc pad", "tool_choice": " none "}, + ) + assert status == 200, body + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "fc pad", "function_call": " auto "}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_modalities_text_as_noop() -> None: + server, thread, port = _server() + try: + for mods in (["text"], [" text "], ["text"]): + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": f"mod {mods!r}", "modalities": mods}, + ) + assert status == 200, (mods, body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_still_rejects_non_text_modalities() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "audio mod", "modalities": ["audio"]}, + ) + assert status == 400, body + assert "invalid_chat_era_field" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_padded_required_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "required pad"}], + "tool_choice": " required ", + }, + ) + assert status == 400, body + assert "invalid_tool_choice" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_padded_tool_choice_none_auto() + test_http_chat_accepts_padded_function_call_none_auto() + test_http_completions_accepts_padded_tool_choice_function_call() + test_http_completions_accepts_modalities_text_as_noop() + test_http_completions_still_rejects_non_text_modalities() + test_http_chat_still_rejects_padded_required_without_tools() + print("ok") diff --git a/tests/test_tool_description_blank_omit_http_honesty.py b/tests/test_tool_description_blank_omit_http_honesty.py new file mode 100644 index 000000000..5b77cf5cd --- /dev/null +++ b/tests/test_tool_description_blank_omit_http_honesty.py @@ -0,0 +1,250 @@ +"""Empty/whitespace tool.function.description is omit-real over HTTP. + +Official OpenAI SDKs serialize an omitted function description as ``""`` +or a whitespace-only string. Leaving that key on the body is not +omit-equivalent: ``proxy_completion`` forwards it and several providers +treat a blank description as a malformed tool. + +These cases assert the buyer-visible contract: + +* chat and Responses return 200 +* mock ``echo.tools`` no longer contains a blank ``description`` +* non-empty descriptions stay on the payload +* non-string descriptions stay ``invalid_tools`` +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + SecurityConfig, + _validate_chat_tools, + build_server, +) + +_TEST_AUTH_TOKEN = "tool_description_blank_omit_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _echo_function(body: dict) -> dict: + echo = body.get("echo") or {} + tools = echo.get("tools") or [] + assert tools, body + function = tools[0].get("function") + assert isinstance(function, dict), body + return function + + +def test_validate_chat_tools_pops_blank_description() -> None: + body = { + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": " \u00a0 ", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + } + validated = _validate_chat_tools(body) + assert validated is not None + function = validated[0]["function"] + assert "description" not in function + assert "description" not in body["tools"][0]["function"] + assert function.get("name") == "lookup" + + +def test_http_chat_omits_empty_tool_description() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty desc"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 200, body + function = _echo_function(body) + assert "description" not in function + assert function.get("parameters") == {"type": "object", "properties": {}} + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_omits_whitespace_tool_description() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "blank desc"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": " \t ", + }, + } + ], + }, + ) + assert status == 200, body + assert "description" not in _echo_function(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_omits_blank_tool_description() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses blank desc", + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "", + "parameters": None, + }, + } + ], + }, + ) + assert status == 200, body + function = _echo_function(body) + assert "description" not in function + assert "parameters" not in function + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_keeps_non_blank_tool_description() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "keep desc"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "find things", + }, + } + ], + }, + ) + assert status == 200, body + assert _echo_function(body).get("description") == "find things" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_description_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc bad"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": ["not", "a", "string"], + }, + } + ], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_validate_chat_tools_pops_blank_description() + test_http_chat_omits_empty_tool_description() + test_http_chat_omits_whitespace_tool_description() + test_http_responses_omits_blank_tool_description() + test_http_chat_keeps_non_blank_tool_description() + test_http_chat_rejects_tool_description_non_string() + print("ok") diff --git a/tests/test_tool_description_length_http_honesty.py b/tests/test_tool_description_length_http_honesty.py new file mode 100644 index 000000000..c76455d8b --- /dev/null +++ b/tests/test_tool_description_length_http_honesty.py @@ -0,0 +1,114 @@ +"""Chat tools function.description max length honesty over HTTP (1024 chars).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "tool_description_length_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_tool_description_at_1024_chars() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc 1024"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "d" * 1024, + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_description_over_1024_chars() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc 1025"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "d" * 1025, + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "1024" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_tool_description_at_1024_chars() + test_http_chat_rejects_tool_description_over_1024_chars() + print("ok") diff --git a/tests/test_tool_description_parameters_null_noop_http_honesty.py b/tests/test_tool_description_parameters_null_noop_http_honesty.py new file mode 100644 index 000000000..547b91e92 --- /dev/null +++ b/tests/test_tool_description_parameters_null_noop_http_honesty.py @@ -0,0 +1,194 @@ +"""tool.function description/parameters null are omit-equivalent over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "tool_description_parameters_null_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_tool_description_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc null"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": None, + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_tool_parameters_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "params null"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": None, + }, + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_tool_description_and_parameters_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "both null"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": None, + "parameters": None, + "strict": None, + }, + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_description_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc bad"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": 123, + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_parameters_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "params bad"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": "not-an-object", + }, + } + ], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_tool_description_null() + test_http_chat_accepts_tool_parameters_null() + test_http_chat_accepts_tool_description_and_parameters_null() + test_http_chat_rejects_tool_description_non_string() + test_http_chat_rejects_tool_parameters_non_object() + print("ok") diff --git a/tests/test_tool_function_name_charset_http_honesty.py b/tests/test_tool_function_name_charset_http_honesty.py new file mode 100644 index 000000000..ee7fc5458 --- /dev/null +++ b/tests/test_tool_function_name_charset_http_honesty.py @@ -0,0 +1,191 @@ +"""tool.function.name charset is fail-closed over HTTP. + +OpenAI function names must match ``[a-zA-Z0-9_-]{1,64}`` — the same +ASCII charset now enforced on ``response_format.json_schema.name``. +``str.isalnum()`` alone accepts Unicode letters and digits (``café``, +``名前``, Arabic-Indic digits). Forwarding those names is not honest: +``proxy_completion`` sends the body and the buyer sees an opaque +provider rejection instead of a named ``invalid_tools`` next action. + +These cases assert the buyer-visible contract: + +* chat and Responses return 400 ``invalid_tools`` for Unicode names +* a 64-character legal ASCII name is kept on mock ``echo.tools`` +* a legal short name is unchanged +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + SecurityConfig, + _validate_chat_tools, + build_server, +) + +_TEST_AUTH_TOKEN = "tool_function_name_charset_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _echo_function(body: dict) -> dict: + echo = body.get("echo") or {} + tools = echo.get("tools") or [] + assert tools, body + function = tools[0].get("function") + assert isinstance(function, dict), body + return function + + +def _function_tool(name: str) -> dict: + return { + "type": "function", + "function": { + "name": name, + "parameters": {"type": "object", "properties": {}}, + }, + } + + +def test_validate_chat_tools_rejects_unicode_function_name() -> None: + """Python str.isalnum() accepts café/名前/١٢٣; OpenAI does not.""" + for illegal_name in ("café", "名前", "lookup_١٢٣"): + body = {"tools": [_function_tool(illegal_name)]} + try: + _validate_chat_tools(body) + except Exception as exc: + assert getattr(exc, "code", None) == "invalid_tools" + assert "must match [a-zA-Z0-9_-]" in str(exc) + continue + raise AssertionError(f"{illegal_name!r} tool.function.name must fail closed") + + +def test_http_chat_rejects_unicode_function_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tool unicode"}], + "tools": [_function_tool("café")], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_unicode_function_name() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses tool unicode", + "tools": [_function_tool("名前")], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_keeps_legal_function_name() -> None: + server, thread, port = _server() + try: + max_name = "B" * 64 + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses tool max", + "tools": [_function_tool(max_name)], + }, + ) + assert status == 200, body + assert _echo_function(body).get("name") == max_name + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_keeps_legal_function_name() -> None: + server, thread, port = _server() + try: + legal_name = "lookup_balance-01" + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tool keep"}], + "tools": [_function_tool(legal_name)], + }, + ) + assert status == 200, body + assert _echo_function(body).get("name") == legal_name + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_validate_chat_tools_rejects_unicode_function_name() + test_http_chat_rejects_unicode_function_name() + test_http_responses_rejects_unicode_function_name() + test_http_responses_keeps_legal_function_name() + test_http_chat_keeps_legal_function_name() + print("ok") diff --git a/tests/test_tool_function_null_fields_pop_http_honesty.py b/tests/test_tool_function_null_fields_pop_http_honesty.py new file mode 100644 index 000000000..945d02f80 --- /dev/null +++ b/tests/test_tool_function_null_fields_pop_http_honesty.py @@ -0,0 +1,328 @@ +"""tool.function null optional fields are omit-real over HTTP. + +SDK optional defaults often serialize omitted fields as JSON null. Accepting +those keys without popping them is not omit-equivalent: ``proxy_completion`` +forwards the body to the provider, and several OpenAI-compatible backends +reject ``parameters: null`` / ``description: null`` as non-objects. + +These cases assert the buyer-visible contract: + +* chat and Responses return 200 +* mock ``echo.tools`` no longer contains the null keys +* non-null wrong types stay fail-closed with named ``invalid_tools`` +* response_format.json_schema.strict null is also omit-real +""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import ( # noqa: E402 + SecurityConfig, + _validate_chat_tools, + build_server, +) + +_TEST_AUTH_TOKEN = "tool_function_null_fields_pop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def _echo_function(body: dict) -> dict: + echo = body.get("echo") or {} + tools = echo.get("tools") or [] + assert tools, body + function = tools[0].get("function") + assert isinstance(function, dict), body + return function + + +def test_validate_chat_tools_pops_null_optional_function_fields() -> None: + body = { + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": None, + "parameters": None, + "strict": None, + }, + } + ] + } + validated = _validate_chat_tools(body) + assert validated is not None + function = validated[0]["function"] + assert "description" not in function + assert "parameters" not in function + assert "strict" not in function + # In-place mutation on the request body (proxy reads the same object). + assert "description" not in body["tools"][0]["function"] + assert "parameters" not in body["tools"][0]["function"] + assert "strict" not in body["tools"][0]["function"] + + +def test_http_chat_omits_tool_description_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc null"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": None, + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 200, body + function = _echo_function(body) + assert "description" not in function + assert function.get("parameters") == {"type": "object", "properties": {}} + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_omits_tool_parameters_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "params null"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": None, + }, + } + ], + }, + ) + assert status == 200, body + function = _echo_function(body) + assert "parameters" not in function + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_omits_tool_description_parameters_and_strict_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "all null"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": None, + "parameters": None, + "strict": None, + }, + } + ], + }, + ) + assert status == 200, body + function = _echo_function(body) + assert "description" not in function + assert "parameters" not in function + assert "strict" not in function + assert function.get("name") == "lookup" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_omits_tool_description_and_parameters_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "responses null tools", + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": None, + "parameters": None, + "strict": None, + }, + } + ], + }, + ) + assert status == 200, body + function = _echo_function(body) + assert "description" not in function + assert "parameters" not in function + assert "strict" not in function + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_description_non_string() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "desc bad"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": 123, + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_parameters_non_object() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "params bad"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": "not-an-object", + }, + } + ], + }, + ) + assert status == 400, body + assert "invalid_tools" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_keeps_non_null_tool_fields() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "keep fields"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "find things", + "parameters": {"type": "object", "properties": {}}, + "strict": True, + }, + } + ], + }, + ) + assert status == 200, body + function = _echo_function(body) + assert function.get("description") == "find things" + assert function.get("parameters") == {"type": "object", "properties": {}} + assert function.get("strict") is True + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_validate_chat_tools_pops_null_optional_function_fields() + test_http_chat_omits_tool_description_null() + test_http_chat_omits_tool_parameters_null() + test_http_chat_omits_tool_description_parameters_and_strict_null() + test_http_responses_omits_tool_description_and_parameters_null() + test_http_chat_rejects_tool_description_non_string() + test_http_chat_rejects_tool_parameters_non_object() + test_http_chat_keeps_non_null_tool_fields() + print("ok") diff --git a/tests/test_tool_strict_null_noop_http_honesty.py b/tests/test_tool_strict_null_noop_http_honesty.py new file mode 100644 index 000000000..a2591bb2c --- /dev/null +++ b/tests/test_tool_strict_null_noop_http_honesty.py @@ -0,0 +1,190 @@ +"""tool.function.strict and json_schema.strict null are omit-equivalent over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "tool_strict_null_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_tool_function_strict_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "strict null"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + "strict": None, + }, + } + ], + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_rejects_tool_function_strict_non_boolean() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "strict bad"}], + "tools": [ + { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + "strict": "yes", + }, + } + ], + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_tools" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_response_format_json_schema_strict_null() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [ + { + "role": "user", + "content": "return json with a name field for schema strict null", + } + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + }, + "strict": None, + }, + }, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_rejects_parallel_tool_calls_true_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "parallel true no tools", + "parallel_tool_calls": True, + }, + ) + assert status == 400, body + blob = json.dumps(body) + assert "invalid_parallel_tool_calls" in blob + assert "unknown_fields" not in blob + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_parallel_tool_calls_false_without_tools() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + { + "model": "mock-planner", + "input": "parallel false no tools", + "parallel_tool_calls": False, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_tool_function_strict_null() + test_http_chat_rejects_tool_function_strict_non_boolean() + test_http_chat_accepts_response_format_json_schema_strict_null() + test_http_responses_rejects_parallel_tool_calls_true_without_tools() + test_http_responses_accepts_parallel_tool_calls_false_without_tools() + print("ok") diff --git a/tests/test_top_logprobs_zero_omit_noop_http_honesty.py b/tests/test_top_logprobs_zero_omit_noop_http_honesty.py new file mode 100644 index 000000000..f362207b3 --- /dev/null +++ b/tests/test_top_logprobs_zero_omit_noop_http_honesty.py @@ -0,0 +1,130 @@ +"""top_logprobs:0 as omit no-op honesty on chat and Completions over HTTP.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "top_logprobs_zero_omit_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_top_logprobs_zero_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tlp zero"}], + "top_logprobs": 0, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_accepts_logprobs_false_top_logprobs_zero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "lp false tlp zero"}], + "logprobs": False, + "top_logprobs": 0, + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_top_logprobs_zero() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "tlp zero", "top_logprobs": 0}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_nonzero_top_logprobs() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "tlp five"}], + "top_logprobs": 5, + }, + ) + assert status == 400, body + assert "invalid_top_logprobs" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_top_logprobs_zero_as_omit() + test_http_chat_accepts_logprobs_false_top_logprobs_zero() + test_http_completions_accepts_top_logprobs_zero() + test_http_chat_still_rejects_nonzero_top_logprobs() + print("ok") diff --git a/tests/test_true_streaming.py b/tests/test_true_streaming.py index ef451955b..7220def82 100644 --- a/tests/test_true_streaming.py +++ b/tests/test_true_streaming.py @@ -118,8 +118,8 @@ def post(payload: dict) -> tuple[str, str]: return response.headers.get("content-type", ""), response.read().decode("utf-8") try: - content_type, sse = post({"messages": [{"role": "user", "content": "stream this"}], "mode": "route", "stream": True}) - _, ref = post({"messages": [{"role": "user", "content": "stream this"}], "mode": "route"}) + content_type, sse = post({"model": "m-model", "messages": [{"role": "user", "content": "stream this"}], "mode": "route", "stream": True}) + _, ref = post({"model": "m-model", "messages": [{"role": "user", "content": "stream this"}], "mode": "route"}) finally: server.shutdown() diff --git a/tests/test_user_null_omit_noop_http_honesty.py b/tests/test_user_null_omit_noop_http_honesty.py new file mode 100644 index 000000000..659627037 --- /dev/null +++ b/tests/test_user_null_omit_noop_http_honesty.py @@ -0,0 +1,195 @@ +"""OpenAI user:null as omit no-op honesty over HTTP (chat/completions/responses/embeddings).""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 + +_TEST_AUTH_TOKEN = "user_null_omit_noop_http_honesty_token" # noqa: S105 + + +def build() -> TaskOrchestrator: + return TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing", "embedding"))] + ) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_TEST_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _server(): + server = build_server( + build(), + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN, rate_limit_requests=10_000), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, server.server_address[1] + + +def test_http_chat_accepts_user_null_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "user null chat"}], + "user": None, + }, + ) + assert status == 200, body + assert body.get("object") == "chat.completion", body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_completions_accepts_user_null_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/completions", + {"model": "mock-planner", "prompt": "user null completions", "user": None}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_responses_accepts_user_null_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/responses", + {"model": "mock-planner", "input": "user null responses", "user": None}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_accepts_user_null_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/embeddings", + {"model": "mock-planner", "input": "user null embeddings", "user": None}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_batch_embeddings_accepts_user_null_as_omit() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/batch/embeddings", + {"model": "mock-planner", "inputs": ["user null batch"], "user": None}, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_empty_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "empty user"}], + "user": " ", + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_rejects_non_string_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "int user"}], + "user": 42, + }, + ) + assert status == 400, body + assert "invalid_user" in json.dumps(body) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_chat_still_accepts_valid_user() -> None: + server, thread, port = _server() + try: + status, body = _post( + port, + "/v1/chat/completions", + { + "model": "mock-planner", + "messages": [{"role": "user", "content": "valid user"}], + "user": "buyer_account_9", + }, + ) + assert status == 200, body + finally: + server.shutdown() + thread.join(timeout=5) + + +if __name__ == "__main__": + test_http_chat_accepts_user_null_as_omit() + test_http_completions_accepts_user_null_as_omit() + test_http_responses_accepts_user_null_as_omit() + test_http_embeddings_accepts_user_null_as_omit() + test_http_batch_embeddings_accepts_user_null_as_omit() + test_http_chat_still_rejects_empty_user() + test_http_chat_still_rejects_non_string_user() + test_http_chat_still_accepts_valid_user() + print("ok")