From aa90cf9678f2c739a946356a7e9a93e0ffdb5d29 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:16:07 +0900 Subject: [PATCH 1/3] fix: preserve caller generation budgets at HTTP ingress --- CHANGELOG.md | 4 + contextual_orchestrator/server.py | 45 ++++-------- ...chat_max_completion_tokens_http_honesty.py | 19 ----- ...est_completions_max_tokens_http_honesty.py | 18 ----- tests/test_generation_token_ingress.py | 73 +++++++++++++++++++ ...esponses_max_output_tokens_http_honesty.py | 19 ----- .../test_responses_max_tokens_http_honesty.py | 18 ----- 7 files changed, 91 insertions(+), 105 deletions(-) create mode 100644 tests/test_generation_token_ingress.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f3f544e59..6b4078cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Generation-token inputs retain positive-integer validation without an + arbitrary shared 1,048,576-token ceiling. Responses normalizes legacy token + aliases to `max_output_tokens` before provider forwarding, preserving caller + budgets and native-field precedence (issue #1151). Model-specific limits remain. - Finite administrator model timeouts now use one end-to-end deadline across local admission, connection, retry, and streamed chunks; synchronous embeddings use the selected model's policy. Timeout values are capped at the diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 0bc13c9ab..324d29859 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -1608,7 +1608,7 @@ def _validate_chat_model(body: dict[str, Any]) -> str: return model def _validate_completions_max_tokens(body: dict[str, Any]) -> int | None: - """Legacy Completions ``max_tokens`` — positive integer capped at 1_048_576.""" + """Validate legacy Completions ``max_tokens`` as a positive integer.""" if "max_tokens" not in body: return None max_tokens = body.get("max_tokens") @@ -1621,17 +1621,11 @@ def _validate_completions_max_tokens(body: dict[str, Any]) -> int | None: return None if max_tokens < 1: raise RequestError(400, "invalid_max_tokens", "max_tokens must be a positive integer") - if max_tokens > 1_048_576: - raise RequestError( - 400, - "invalid_max_tokens", - "max_tokens must be at most 1048576", - ) body["max_tokens"] = max_tokens return max_tokens def _validate_chat_max_completion_tokens(body: dict[str, Any]) -> int | None: - """Chat Completions ``max_completion_tokens`` — positive integer capped at 1_048_576. + """Validate Chat Completions ``max_completion_tokens`` as a positive integer. OpenAI prefers this over legacy ``max_tokens`` for chat. When both are set, ``max_completion_tokens`` wins so clients get a single honest budget. @@ -1652,12 +1646,6 @@ def _validate_chat_max_completion_tokens(body: dict[str, Any]) -> int | None: "invalid_max_completion_tokens", "max_completion_tokens must be a positive integer", ) - if max_completion_tokens > 1_048_576: - raise RequestError( - 400, - "invalid_max_completion_tokens", - "max_completion_tokens must be at most 1048576", - ) body["max_completion_tokens"] = max_completion_tokens return max_completion_tokens @@ -1668,32 +1656,28 @@ def _validate_responses_max_output_tokens(body: dict[str, Any]) -> int | None: Official Responses clients send ``max_output_tokens`` rather than chat-era ``max_tokens``. Accept and type-check so the field is not opaque ``unknown_fields``; value is left on the body for provider passthrough. - Cap matches ``max_tokens`` (1_048_576). Digit strings and whole-number - floats (JS JSON) coerce. + Normalize aliases with precedence: native, completion, then legacy tokens. + Digit strings and whole-number floats (JS JSON) coerce. """ - if "max_output_tokens" not in body: - return None - value = _coerce_optional_int( + output_token_limit = _coerce_optional_int( body.get("max_output_tokens"), error_code="invalid_max_output_tokens", message="max_output_tokens must be a positive integer", ) - if value is None: + if output_token_limit is None: + output_token_limit = _validate_chat_max_completion_tokens(body) + if output_token_limit is None: + output_token_limit = _validate_completions_max_tokens(body) + if output_token_limit is None: return None - body["max_output_tokens"] = value - if value < 1: + body["max_output_tokens"] = output_token_limit + if output_token_limit < 1: raise RequestError( 400, "invalid_max_output_tokens", "max_output_tokens must be a positive integer", ) - if value > 1_048_576: - raise RequestError( - 400, - "invalid_max_output_tokens", - "max_output_tokens must be at most 1048576", - ) - return value + return output_token_limit @@ -7630,8 +7614,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di _validate_completions_max_tokens(body) if "max_completion_tokens" in body: _validate_chat_max_completion_tokens(body) - if "max_output_tokens" in body: - _validate_responses_max_output_tokens(body) + _validate_responses_max_output_tokens(body) if "max_tool_calls" in body: _validate_responses_max_tool_calls(body) _validate_openai_sdk_control_fields(body, endpoint_path="/v1/responses") diff --git a/tests/test_chat_max_completion_tokens_http_honesty.py b/tests/test_chat_max_completion_tokens_http_honesty.py index 5cfc42dcf..4e52ce5f7 100644 --- a/tests/test_chat_max_completion_tokens_http_honesty.py +++ b/tests/test_chat_max_completion_tokens_http_honesty.py @@ -107,24 +107,6 @@ def test_http_chat_rejects_max_completion_tokens_bool() -> None: thread.join(timeout=5) -def test_http_chat_rejects_max_completion_tokens_too_large() -> None: - server, thread, port = _server() - try: - status, body = _post( - port, - "/v1/chat/completions", - { - "model": "mock-planner", - "messages": [{"role": "user", "content": "huge budget"}], - "max_completion_tokens": 1_048_577, - }, - ) - assert status == 400, body - assert "invalid_max_completion_tokens" in json.dumps(body) - finally: - server.shutdown() - thread.join(timeout=5) - def test_http_chat_prefers_max_completion_tokens_when_both_set() -> None: """When both budgets are present, request must still succeed (max_completion wins).""" @@ -187,7 +169,6 @@ def test_http_chat_accepts_max_completion_tokens_omitted() -> None: test_http_chat_accepts_max_completion_tokens() test_http_chat_rejects_max_completion_tokens_zero() test_http_chat_rejects_max_completion_tokens_bool() - test_http_chat_rejects_max_completion_tokens_too_large() test_http_chat_prefers_max_completion_tokens_when_both_set() test_http_chat_rejects_invalid_max_tokens_when_only_legacy() test_http_chat_accepts_max_completion_tokens_omitted() diff --git a/tests/test_completions_max_tokens_http_honesty.py b/tests/test_completions_max_tokens_http_honesty.py index 4f6e6870e..5f938de53 100644 --- a/tests/test_completions_max_tokens_http_honesty.py +++ b/tests/test_completions_max_tokens_http_honesty.py @@ -64,7 +64,6 @@ def test_http_max_tokens_applies_and_restores() -> None: server.shutdown() thread.join(timeout=5) - def test_http_rejects_non_positive_max_tokens() -> None: server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -116,20 +115,3 @@ def test_http_rejects_bool_max_tokens() -> None: finally: server.shutdown() thread.join(timeout=5) - - -def test_http_rejects_oversized_max_tokens() -> None: - server = build_server(build(), port=0, security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN)) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - port = server.server_address[1] - try: - status, body = _post( - port, - {"model": "mock-planner", "prompt": "hello", "max_tokens": 2_000_000}, - ) - assert status == 400, body - assert body["error"]["code"] == "invalid_max_tokens" - finally: - server.shutdown() - thread.join(timeout=5) diff --git a/tests/test_generation_token_ingress.py b/tests/test_generation_token_ingress.py new file mode 100644 index 000000000..3e13af4f6 --- /dev/null +++ b/tests/test_generation_token_ingress.py @@ -0,0 +1,73 @@ +"""Generation limits remain caller values at the model execution boundary.""" + +import http.client +import json +import threading + +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.orchestrator import ModelClient +from contextual_orchestrator.server import SecurityConfig, build_server + + +@pytest.mark.parametrize( + "endpoint_path, budget_field, input_fields", + [ + ("/v1/completions", "max_tokens", {"prompt": "hello"}), + ("/v1/completions", "max_completion_tokens", {"prompt": "hello"}), + ("/v1/chat/completions", "max_tokens", {"messages": [{"role": "user", "content": "hello"}]}), + ("/v1/chat/completions", "max_completion_tokens", {"messages": [{"role": "user", "content": "hello"}]}), + ("/v1/responses", "max_tokens", {"input": "hello"}), + ("/v1/responses", "max_completion_tokens", {"input": "hello"}), + ("/v1/responses", "max_output_tokens", {"input": "hello"}), + ("/v1/responses", "max_output_tokens", {"input": "hello", "max_completion_tokens": 32, "max_tokens": 16}), + ("/v1/responses", "max_completion_tokens", {"input": "hello", "max_output_tokens": None, "max_tokens": 16}), + ("/v1/responses", "max_tokens", {"input": "hello", "max_output_tokens": None, "max_completion_tokens": None}), + ], +) +@pytest.mark.parametrize("requested_limit", [64, 1_048_577]) +def test_http_preserves_caller_generation_limit(monkeypatch, endpoint_path, budget_field, input_fields, requested_limit): + """A boundary fixture above the former cap does not claim real model capacity.""" + model_client = ModelClient() + observed_limits = [] + original_mock = model_client._mock + original_raw_mock = model_client._mock_raw + + def observe_mock(model_agent, *call_args, **call_kwargs): + observed_limits.append(model_client.effective_max_output_tokens(model_agent)) + return original_mock(model_agent, *call_args, **call_kwargs) + + def observe_raw_mock(model_agent, provider_endpoint, provider_payload): + assert provider_endpoint.strip("/") == "responses" + observed_limits.append(provider_payload.get("max_output_tokens")) + return original_raw_mock(model_agent, provider_endpoint, provider_payload) + + monkeypatch.setattr(model_client, "_mock", observe_mock) + monkeypatch.setattr(model_client, "_mock_raw", observe_raw_mock) + task_orchestrator = TaskOrchestrator( + [ModelAgent("general_agent", "mock-planner", tags=("reasoning", "writing"))], + client=model_client, + ) + http_server = build_server( + task_orchestrator, port=0, security=SecurityConfig(auth_token="fixture-token") + ) + server_thread = threading.Thread(target=http_server.serve_forever, daemon=True) + server_thread.start() + http_connection = http.client.HTTPConnection(*http_server.server_address, timeout=10) + try: + request_payload = {"model": "mock-planner", **input_fields, budget_field: requested_limit} + http_connection.request( + "POST", endpoint_path, json.dumps(request_payload), + {"Content-Type": "application/json", "Authorization": "Bearer fixture-token"}, + ) + http_response = http_connection.getresponse() + response_payload = json.loads(http_response.read()) + assert http_response.status == 200, response_payload + assert observed_limits and all(observed_limit == requested_limit for observed_limit in observed_limits), observed_limits + finally: + http_connection.close() + http_server.shutdown() + server_thread.join(timeout=5) + http_server.server_close() + task_orchestrator.close() diff --git a/tests/test_responses_max_output_tokens_http_honesty.py b/tests/test_responses_max_output_tokens_http_honesty.py index 5bf63db61..aa8b8b0bd 100644 --- a/tests/test_responses_max_output_tokens_http_honesty.py +++ b/tests/test_responses_max_output_tokens_http_honesty.py @@ -68,7 +68,6 @@ def test_http_responses_accepts_valid_max_output_tokens() -> None: server.shutdown() thread.join(timeout=5) - def test_http_responses_accepts_omit_max_output_tokens() -> None: server, thread, port = _server() try: @@ -134,21 +133,3 @@ def test_http_responses_rejects_boolean_max_output_tokens() -> None: finally: server.shutdown() thread.join(timeout=5) - - -def test_http_responses_rejects_oversize_max_output_tokens() -> None: - server, thread, port = _server() - try: - status, body = _post( - port, - { - "model": "mock-planner", - "input": "huge budget", - "max_output_tokens": 2_000_000, - }, - ) - assert status == 400, body - assert "invalid_max_output_tokens" in json.dumps(body) - finally: - server.shutdown() - thread.join(timeout=5) diff --git a/tests/test_responses_max_tokens_http_honesty.py b/tests/test_responses_max_tokens_http_honesty.py index 2dc16f85b..60d87f4d2 100644 --- a/tests/test_responses_max_tokens_http_honesty.py +++ b/tests/test_responses_max_tokens_http_honesty.py @@ -106,23 +106,6 @@ def test_http_responses_rejects_non_integer_max_tokens() -> None: thread.join(timeout=5) -def test_http_responses_rejects_oversize_max_completion_tokens() -> None: - server, thread, port = _server() - try: - status, body = _post( - port, - { - "model": "mock-planner", - "input": "hi", - "max_completion_tokens": 2_000_000, - }, - ) - assert status == 400, body - assert "invalid_max_completion_tokens" in json.dumps(body) - finally: - server.shutdown() - thread.join(timeout=5) - def test_http_responses_rejects_boolean_max_completion_tokens() -> None: server, thread, port = _server() @@ -147,6 +130,5 @@ def test_http_responses_rejects_boolean_max_completion_tokens() -> None: test_http_responses_accepts_valid_max_completion_tokens() test_http_responses_rejects_zero_max_tokens() test_http_responses_rejects_non_integer_max_tokens() - test_http_responses_rejects_oversize_max_completion_tokens() test_http_responses_rejects_boolean_max_completion_tokens() print("ok") From 53292674186c6a5052defde8662dc01fd12ea502 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 01:32:35 +0900 Subject: [PATCH 2/3] test: assert canonical Responses provider budgets --- tests/test_generation_token_ingress.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_generation_token_ingress.py b/tests/test_generation_token_ingress.py index 3e13af4f6..d8b00a4a0 100644 --- a/tests/test_generation_token_ingress.py +++ b/tests/test_generation_token_ingress.py @@ -35,11 +35,15 @@ def test_http_preserves_caller_generation_limit(monkeypatch, endpoint_path, budg original_raw_mock = model_client._mock_raw def observe_mock(model_agent, *call_args, **call_kwargs): + """Record the budget effective at a mocked model invocation.""" observed_limits.append(model_client.effective_max_output_tokens(model_agent)) return original_mock(model_agent, *call_args, **call_kwargs) def observe_raw_mock(model_agent, provider_endpoint, provider_payload): + """Check the canonical Responses budget at the provider boundary.""" assert provider_endpoint.strip("/") == "responses" + assert "max_tokens" not in provider_payload + assert "max_completion_tokens" not in provider_payload observed_limits.append(provider_payload.get("max_output_tokens")) return original_raw_mock(model_agent, provider_endpoint, provider_payload) From bfb7e2be7304873ecae650e8ab03553babbfe74e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 16:26:18 +0900 Subject: [PATCH 3/3] docs(adr): record removal of unsourced generation-token ceiling Co-authored-by: Cursor --- CHANGELOG.md | 2 +- ...move-unsourced-generation-token-ceiling.md | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 docs/planning/adrs/0132-remove-unsourced-generation-token-ceiling.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b4078cae..6009b8cc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - Generation-token inputs retain positive-integer validation without an arbitrary shared 1,048,576-token ceiling. Responses normalizes legacy token aliases to `max_output_tokens` before provider forwarding, preserving caller - budgets and native-field precedence (issue #1151). Model-specific limits remain. + budgets and native-field precedence (issue #1151; ADR 0132). Model-specific limits remain. - Finite administrator model timeouts now use one end-to-end deadline across local admission, connection, retry, and streamed chunks; synchronous embeddings use the selected model's policy. Timeout values are capped at the diff --git a/docs/planning/adrs/0132-remove-unsourced-generation-token-ceiling.md b/docs/planning/adrs/0132-remove-unsourced-generation-token-ceiling.md new file mode 100644 index 000000000..0eb8dab52 --- /dev/null +++ b/docs/planning/adrs/0132-remove-unsourced-generation-token-ceiling.md @@ -0,0 +1,76 @@ +--- +id: "0132" +title: "Remove the unsourced shared generation-token ceiling" +status: accepted +proposed_date: "2026-09-17" +accepted_date: "2026-09-17" +deciders: + - "repository maintainer" +affected_components: + - "contextual_orchestrator/server.py" +related: + - path: "docs/planning/adrs/0040-streamed-responses-usage-boundary.md" + relation: related +success_criteria: + - metric: "no shared ingress ceiling" + target: "Chat, Completions, and Responses accept positive generation budgets without a gateway-wide numeric hard cap" + source: "tests/test_generation_token_ingress.py" + - metric: "Responses alias honesty" + target: "Responses normalizes max_output_tokens over max_completion_tokens over max_tokens and forwards the canonical field" + source: "tests/test_generation_token_ingress.py" + - metric: "model-specific limits remain" + target: "provider-published agent.max_output_tokens and client effective caps still bound generation" + source: "existing model-budget and client-boundary tests" +--- + +# Remove the unsourced shared generation-token ceiling + +## Context + +Ingress validators for Completions `max_tokens`, Chat +`max_completion_tokens`, and Responses `max_output_tokens` rejected any +positive budget above `1_048_576`. That number entered the tree as a +"practical gateway hard ceiling" without a cited provider catalog limit, +operator policy, or verified product requirement (issue #1151). HTTP tests +proved rejection, not provenance. Numerical execution limits in this gateway +must derive from a provider/model constraint, an approved operator policy, or +a verified product requirement. + +Separately, Responses accepted the legacy aliases but did not always keep a +canonical `max_output_tokens` value on the body used for provider forwarding, +so caller budgets could disappear after validation. + +## Decision + +Remove the shared `1_048_576` ingress ceiling. Keep positive-integer (and +coercion) validation. Keep model-specific and client `effective_max_output_tokens` +enforcement. Do not replace the removed number with another arbitrary +gateway-wide constant, and do not silently clamp. + +On `/v1/responses`, normalize generation budgets with precedence +`max_output_tokens` > `max_completion_tokens` > `max_tokens` (falling through +nulls) onto `max_output_tokens` before provider forwarding. + +## Consequences + +- Callers may request generation budgets larger than the former shared ceiling; + providers and model metadata still reject or bound physically unsupported + sizes. +- Operators who need a gateway-wide cap must introduce an explicit, sourced + policy (KV/operator config or catalog-derived limit), not an unsourced + constant in the validator. +- Regression coverage uses a boundary fixture above the former cap only to + prove passthrough; it does not claim any live model supports that size. + +## References + +OpenAI. (n.d.). *Create a model response | OpenAI API reference* +(`max_output_tokens`). Retrieved September 17, 2026, from +https://platform.openai.com/docs/api-reference/responses/create + +OpenAI. (n.d.). *Chat Completions | OpenAI API reference* +(`max_completion_tokens`, `max_tokens`). Retrieved September 17, 2026, from +https://platform.openai.com/docs/api-reference/chat/create + +Issue #1151 — Validate or replace the unsubstantiated common generation-token +ceiling.