From 3daf0a70f5e6e69c9dfad8dd27cbb1b318d5e4e8 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:04:31 -0700 Subject: [PATCH] fix(image_gen): Codex-auth images use the native images endpoints, no chat host model The openai-codex image provider rode a Responses call with a hosted image_generation tool on a pinned chat model (gpt-5.5). Two failure classes came with that shape: when OpenAI withdrew gpt-5.5 from an account cohort every image call 404'd while chat kept working (#105398, #107076), and the host model was free to answer in text instead of calling the tool, so we streamed SSE, kept partial frames and retried on empty streams. Post to chatgpt.com/backend-api/codex/images/generations and images/edits instead - the route the official Codex client uses (codex-rs/ext/image-generation). No host model, no SSE, no partial-frame handling; the response is a plain JSON body with b64_json. Remote source URLs are fetched client-side and inlined as data URLs because the backend's own downloader 400s on ordinary public images. The backend treats model/quality/size as advisory (#107233), so the result now reports reported_quality/reported_size next to the requested values plus the x-codex-imagegen-request-id for support. GPT Image 2.5 is deliberately not added to this catalog: the backend accepts any model id, including nonexistent ones, and generates with its server-managed engine (C2PA reports gpt-image 2.0), so a 2.5 tier here would be a label with no effect (#106708). --- plugins/image_gen/openai-codex/__init__.py | 292 ++++-------- plugins/image_gen/openai-codex/plugin.yaml | 2 +- .../image_gen/test_openai_codex_provider.py | 420 ++++-------------- .../image-gen-provider-plugin.md | 2 +- .../user-guide/features/image-generation.md | 32 +- 5 files changed, 200 insertions(+), 548 deletions(-) diff --git a/plugins/image_gen/openai-codex/__init__.py b/plugins/image_gen/openai-codex/__init__.py index 1bcee0ae3b3ab..f84dcc433d4d0 100644 --- a/plugins/image_gen/openai-codex/__init__.py +++ b/plugins/image_gen/openai-codex/__init__.py @@ -1,13 +1,18 @@ """OpenAI image generation — ChatGPT/Codex OAuth variant. -Same catalog/tiers as the ``openai`` plugin (``gpt-image-2`` low/medium/high), routed -through the Codex Responses API ``image_generation`` tool, so no ``OPENAI_API_KEY`` is -needed. Output is PNG; source images travel as Responses ``input_image`` parts. - -Do NOT reintroduce an "account capability" classifier keyed on ``Tool choice -'image_generation' not found in 'tools' parameter``: that 400 is a request-shape -rejection for every account, fixed by omitting tool_choice (``_build_responses_payload``); -any remaining HTTP error must surface verbatim. +Same catalog/tiers as the ``openai`` plugin (``gpt-image-2`` low/medium/high), posted to the +Codex backend's native ``images/generations`` and ``images/edits`` endpoints, the same route the +official Codex client uses (``codex-rs/ext/image-generation``). No ``OPENAI_API_KEY`` is needed. + +There is deliberately NO chat/host model here. An earlier version rode a Responses call with a +hosted ``image_generation`` tool on a pinned chat model (``gpt-5.5``): when OpenAI withdrew that id +from an account cohort every image call 404'd while chat kept working (#105398, #107076), and the +host model was free to answer in text instead of calling the tool. The native route has neither +failure mode. + +The backend does not enforce ``model``/``quality``/``size`` — it accepts unknown model ids and +returns its own quality/size (#107233). We send the catalog values and report what came back +(``reported_quality``/``reported_size``) so a request that was not honoured is diagnosable. """ from __future__ import annotations @@ -16,6 +21,7 @@ import json import logging import os +import uuid from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -27,34 +33,14 @@ logger = logging.getLogger(__name__) -# NOTE: do NOT reintroduce an "account capability" classifier keyed on ``Tool choice 'image_generation' not -# found in 'tools' parameter``. That HTTP 400 is a *request-shape* rejection (the Codex backend resolves -# tool_choice as a function-tool name and never recognizes hosted-tool entries) — it is emitted for every -# account, including accounts where image generation works. A previous version of this file translated that -# 400 into "Image generation is not enabled for the current Codex account. Switch the image provider to -# OpenAI API key, FAL, or xAI.", which reported a universal bug in our own payload as the user's entitlement -# problem and sent people away from a provider that was never actually tried. The request-shape bug is fixed -# by omitting tool_choice (see ``_build_responses_payload``); any remaining HTTP error must surface verbatim -# so it stays diagnosable. See issues #19505, #49008 and #31335. -_MAX_ERROR_BODY_CHARS = 500 - -# Hosts the ``image_generation`` tool call; ``API_MODEL`` does the image work. -_CODEX_CHAT_MODEL = "gpt-5.5" _CODEX_BASE_URL = "https://chatgpt.com/backend-api/codex" -_CODEX_INSTRUCTIONS = ( - "You are an assistant that must fulfill image generation and image editing " - "requests by using the image_generation tool when provided.") +_MAX_ERROR_BODY_CHARS = 500 _MAX_REFERENCE_IMAGES = 16 _MAX_INPUT_IMAGE_BYTES = 25 * 1024 * 1024 -# ``input_image`` accepts raster only; the shared sniffer also knows SVG/TIFF/ICO, which the API rejects. +# The edit endpoint accepts raster only; the shared sniffer also knows SVG/TIFF/ICO, which it rejects. _ACCEPTED_INPUT_MIME = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"}) -# Progressive frames (partial_image_b64) saved as finals produced the "smear" failure mode: -# request 0 partials, never let a partial overwrite a final, only deliver source=final. -_PARTIAL_IMAGES_REQUESTED = 0 -_NONFINAL_RETRIES = 1 # content-agnostic retries when the stream yields no final result - _NO_AUTH = ( "No Codex/ChatGPT OAuth credentials available. Run " "`hermes auth codex` (or `hermes setup` → Codex) to sign in.") @@ -129,17 +115,24 @@ def _data_url_to_input_image_url(value: str) -> str: "Image data URL does not contain supported image bytes") +def _remote_image_to_data_url(value: str) -> str: + """The edit endpoint takes inline data URLs only (as the official client sends), so fetch.""" + import httpx + + response = httpx.get(value, timeout=60.0, follow_redirects=True) + response.raise_for_status() + return _encode_input_image( + response.content, + f"Image URL exceeds 25MB cap: {value}", + f"Image URL did not return a supported image: {value}") + + def _local_image_to_data_url(value: str) -> str: - try: - from agent.file_safety import get_read_block_error + from agent.file_safety import get_read_block_error - blocked = get_read_block_error(value) - if blocked: - raise ValueError(blocked) - except ValueError: - raise - except Exception as exc: - logger.debug("Codex image input read guard unavailable: %s", exc) + blocked = get_read_block_error(value) + if blocked: + raise ValueError(blocked) path = Path(os.path.expanduser(value)).resolve() if not path.is_file(): raise ValueError(f"Image input path does not exist or is not a file: {value}") @@ -151,85 +144,71 @@ def _local_image_to_data_url(value: str) -> str: f"Image input path is not a supported image: {value}") -def _to_input_image_part(value: str) -> Dict[str, str]: - """Convert a URL/data URL/local path into a Responses input_image part.""" +def _to_input_image(value: str) -> Dict[str, str]: + """Convert a URL/data URL/local path into an ``images[]`` entry for ``images/edits``.""" candidate = (value or "").strip() if not candidate: raise ValueError("Blank image input") lowered = candidate.lower() if lowered.startswith(("http://", "https://")): - image_url = candidate + image_url = _remote_image_to_data_url(candidate) elif lowered.startswith("data:"): image_url = _data_url_to_input_image_url(candidate) else: image_url = _local_image_to_data_url(candidate) - return {"type": "input_image", "image_url": image_url} + return {"image_url": image_url} def _normalize_input_images( image_url: Optional[str], reference_image_urls: Optional[List[str]] ) -> List[Dict[str, str]]: values = collect_source_images(image_url, reference_image_urls, limit=_MAX_REFERENCE_IMAGES) - return [_to_input_image_part(value) for value in values] + return [_to_input_image(value) for value in values] -def _build_responses_payload( +def _build_image_request( *, prompt: str, size: str, quality: str, input_images: Optional[List[Dict[str, str]]] = None -) -> Dict[str, Any]: - """Responses body for an image_generation call. No ``tool_choice``: Codex rejects every shape - for forcing the hosted tool (looks it up as a *function* name), so the host model decides, - nudged by ``instructions``.""" - content: List[Dict[str, Any]] = [{"type": "input_text", "text": prompt}, *(input_images or [])] - return { - "model": _CODEX_CHAT_MODEL, - "store": False, - "instructions": _CODEX_INSTRUCTIONS, - "input": [{"type": "message", "role": "user", "content": content}], - "tools": [{ - "type": "image_generation", - "model": API_MODEL, - "size": size, - "quality": quality, - "output_format": "png", - "background": "opaque", - "partial_images": _PARTIAL_IMAGES_REQUESTED, - }], - # No ``tool_choice`` is sent: the chatgpt.com/backend-api/codex backend rejects every shape we have - # for forcing the hosted ``image_generation`` tool. ``{"type": "allowed_tools", "mode": "required", - # "tools": [{"type": "image_generation"}]}`` (and the simpler ``{"type": "image_generation"}`` form) - # both 400 with ``Tool choice 'image_generation' not found in 'tools' parameter`` — the backend - # looks up tool_choice as a *function* name and never recognizes hosted-tool entries. Letting the - # host model decide is the only shape Codex currently accepts; the ``instructions`` above are what - # nudge it toward the tool. See issue #19505. - "stream": True, +) -> Tuple[str, Dict[str, Any]]: + """``(endpoint_path, json_body)`` — ``images/edits`` when sources are present, else + ``images/generations``. Field set mirrors the official client's ``ImageGenerationRequest`` / + ``ImageEditRequest``.""" + body: Dict[str, Any] = { + "prompt": prompt, "model": API_MODEL, "n": 1, "quality": quality, "size": size, + "background": "opaque", } + if input_images: + body["images"] = input_images + return "images/edits", body + return "images/generations", body -def _extract_image_candidates(value: Any) -> Tuple[Optional[str], Optional[str]]: - """``(final_result_b64, latest_partial_b64)`` from a payload tree; a partial never overwrites - a final.""" - result_b64: Optional[str] = None - partial_b64: Optional[str] = None - - def walk(node: Any) -> None: - nonlocal result_b64, partial_b64 - if isinstance(node, dict): - result = node.get("result") if node.get("type") == "image_generation_call" else None - if isinstance(result, str) and result: - result_b64 = result - partial = node.get("partial_image_b64") - if isinstance(partial, str) and partial: - partial_b64 = partial - for child in node.values() if isinstance(node, dict) else node if isinstance(node, list) else (): - walk(child) - - walk(value) - return result_b64, partial_b64 - +def _post_image_request( + token: str, *, prompt: str, size: str, quality: str, input_images: Optional[List[Dict[str, str]]] = None +) -> Dict[str, Any]: + """POST to the native Codex images endpoint; return the decoded JSON body plus + ``imagegen_request_id`` (backend correlation id, for support tickets).""" + import httpx + from agent.codex_headers import codex_cloudflare_headers -def _extract_image_b64(value: Any) -> Optional[str]: - """Image b64 from a payload, preferring a final result over a partial.""" - return next((b64 for b64 in _extract_image_candidates(value) if b64), None) + headers = codex_cloudflare_headers(token) + headers.update({ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "x-codex-image-turn-id": str(uuid.uuid4()), + }) + path, body = _build_image_request(prompt=prompt, size=size, quality=quality, input_images=input_images) + timeout = httpx.Timeout(300.0, connect=30.0, read=300.0, write=60.0, pool=30.0) + with httpx.Client(timeout=timeout, headers=headers) as http: + response = http.post(f"{_CODEX_BASE_URL}/{path}", json=body) + if response.status_code >= 400: + raise RuntimeError( + f"Codex images API returned HTTP {response.status_code}: " + f"{_summarize_error_body(response.text)}") + payload = response.json() + if not isinstance(payload, dict): + raise RuntimeError("Codex images API returned a non-object body") + payload["imagegen_request_id"] = response.headers.get("x-codex-imagegen-request-id") + return payload def _png_pixel_size(raw: bytes) -> Optional[str]: @@ -242,81 +221,6 @@ def _png_pixel_size(raw: bytes) -> Optional[str]: return f"{width}x{height}" -def _iter_sse_json(response: Any): - """JSON payloads from an SSE response, without SDK parsing (events newer than the SDK still parse).""" - event_name: Optional[str] = None - data_lines: List[str] = [] - - def flush(): - nonlocal event_name, data_lines - if not data_lines: - event_name = None - return None - raw = "\n".join(data_lines).strip() - event, event_name, data_lines = event_name, None, [] - if not raw or raw == "[DONE]": - return None - payload = json.loads(raw) - if isinstance(payload, dict) and event and "type" not in payload: - payload["type"] = event - return payload - - for line in response.iter_lines(): - if isinstance(line, bytes): - line = line.decode("utf-8", errors="replace") - line = str(line) - if line == "": - payload = flush() - if payload is not None: - yield payload - elif line.startswith("event:"): - event_name = line[len("event:"):].strip() - elif line.startswith("data:"): - data_lines.append(line[len("data:"):].lstrip()) - payload = flush() - if payload is not None: - yield payload - - -def _collect_image_b64( - token: str, *, prompt: str, size: str, quality: str, input_images: Optional[List[Dict[str, str]]] = None -) -> Optional[Dict[str, str]]: - """Stream a Codex Responses image_generation call → ``{"b64", "source": "final"|"partial"}`` or - ``None``. A partial is kept only when no final arrives; callers must not treat it as success.""" - import httpx - from agent.codex_headers import codex_cloudflare_headers - - headers = codex_cloudflare_headers(token) - headers.update({ - "Accept": "text/event-stream", - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }) - payload = _build_responses_payload( - prompt=prompt, size=size, quality=quality, input_images=input_images) - timeout = httpx.Timeout(300.0, connect=30.0, read=300.0, write=30.0, pool=30.0) - - final_b64: Optional[str] = None - partial_b64: Optional[str] = None - with httpx.Client(timeout=timeout, headers=headers) as http: - with http.stream("POST", f"{_CODEX_BASE_URL}/responses", json=payload) as response: - try: - response.raise_for_status() - except httpx.HTTPStatusError as exc: - exc.response.read() - raise RuntimeError( - f"Codex Responses API returned HTTP {exc.response.status_code}: " - f"{_summarize_error_body(exc.response.text)}" - ) from exc - for event in _iter_sse_json(response): - result_b64, event_partial = _extract_image_candidates(event) - final_b64 = result_b64 or final_b64 - partial_b64 = event_partial or partial_b64 - if final_b64: - return {"b64": final_b64, "source": "final"} - return {"b64": partial_b64, "source": "partial"} if partial_b64 else None - - class OpenAICodexImageGenProvider(StaticImageGenProvider): """gpt-image-2 routed through ChatGPT/Codex OAuth instead of an API key.""" @@ -362,52 +266,22 @@ def generate( tier_id, meta = _resolve_model() size = size_for(aspect) fail = error_factory("openai-codex", aspect, model=tier_id, prompt=prompt) - attempts = _NONFINAL_RETRIES + 1 try: input_images = _normalize_input_images(image_url, reference_image_urls) except Exception as exc: return fail(f"Invalid image input for Codex image editing: {exc}", "invalid_image_input") try: - collected: Optional[Dict[str, str]] = None - for attempt in range(attempts): - collected = _collect_image_b64( - token, prompt=prompt, size=size, quality=meta["quality"], - input_images=input_images or None) - if collected and collected.get("source") == "final" and collected.get("b64"): - break - if attempt < _NONFINAL_RETRIES: - partial = collected and collected.get("source") == "partial" - logger.warning( - "Codex image stream ended with %s (attempt %s/%s); " - "retrying once before failing closed.", - "progressive-only partial frame" if partial else "no image_generation_call result", - attempt + 1, attempts) + payload = _post_image_request( + token, prompt=prompt, size=size, quality=meta["quality"], input_images=input_images or None) except Exception as exc: logger.debug("Codex image generation failed", exc_info=True) return fail(f"OpenAI image generation via Codex auth failed: {exc}", "api_error") - if not collected or not collected.get("b64"): - return fail( - f"Codex response contained no image_generation_call result after {attempts} attempt(s)", - "empty_response") - image_source = collected.get("source") or "unknown" - b64 = collected["b64"] - # Never deliver a progressive-only frame as success (smeared previews). - if image_source != "final": - try: - pixel_hint = _png_pixel_size(base64.b64decode(b64, validate=False)) - except Exception: - pixel_hint = None - detail = ( - "Codex returned only a progressive partial image frame after " - f"{attempts} attempt(s); refusing to save it as a final deliverable.") - if pixel_hint: - detail = f"{detail} partial_pixel_size={pixel_hint}." - return { - **fail(detail, "incomplete_image"), "image_source": image_source, "requested_size": size, - "partial_pixel_size": pixel_hint, "nonfinal_retries": _NONFINAL_RETRIES, - } + data = payload.get("data") + b64 = data[0].get("b64_json") if isinstance(data, list) and data and isinstance(data[0], dict) else None + if not isinstance(b64, str) or not b64: + return fail("Codex images API response contained no image data", "empty_response") try: pixel_size = _png_pixel_size(base64.b64decode(b64)) @@ -419,7 +293,9 @@ def generate( provider="openai-codex", modality="image" if input_images else "text", extra={ "size": size, "quality": meta["quality"], "input_image_count": len(input_images), - "image_source": image_source, "requested_size": size, "pixel_size": pixel_size, + "requested_size": size, "pixel_size": pixel_size, + "reported_quality": payload.get("quality"), "reported_size": payload.get("size"), + "imagegen_request_id": payload.get("imagegen_request_id"), }) diff --git a/plugins/image_gen/openai-codex/plugin.yaml b/plugins/image_gen/openai-codex/plugin.yaml index 61757773e19c8..b90db889e6725 100644 --- a/plugins/image_gen/openai-codex/plugin.yaml +++ b/plugins/image_gen/openai-codex/plugin.yaml @@ -1,5 +1,5 @@ name: openai-codex version: 1.0.0 -description: "OpenAI image generation backed by ChatGPT/Codex OAuth (gpt-image-2 via the Responses image_generation tool). Saves generated images to $HERMES_HOME/cache/images/." +description: "OpenAI image generation backed by ChatGPT/Codex OAuth (gpt-image-2 via the native Codex images/generations and images/edits endpoints). Saves generated images to $HERMES_HOME/cache/images/." author: NousResearch kind: backend diff --git a/tests/plugins/image_gen/test_openai_codex_provider.py b/tests/plugins/image_gen/test_openai_codex_provider.py index 244b720bf18c1..89a7832bbfaec 100644 --- a/tests/plugins/image_gen/test_openai_codex_provider.py +++ b/tests/plugins/image_gen/test_openai_codex_provider.py @@ -1,17 +1,18 @@ """Tests for the bundled ``openai-codex`` image_gen plugin. -Mirrors ``test_openai_provider.py`` but targets the standalone -Codex/ChatGPT-OAuth-backed provider that uses the Responses -``image_generation`` tool path instead of the ``images.generate`` REST -endpoint. +Mirrors ``test_openai_provider.py`` but targets the ChatGPT-OAuth-backed provider that posts to +the Codex backend's native ``images/generations`` / ``images/edits`` endpoints (the route the +official Codex client uses) — no chat host model, no hosted-tool SSE stream (#105398, #107076). """ from __future__ import annotations +import base64 import importlib import json from pathlib import Path +import httpx import pytest # The plugin directory uses a hyphen, which is not a valid Python identifier @@ -28,14 +29,18 @@ ) +def _png_bytes() -> bytes: + return bytes.fromhex(_PNG_HEX) + + def _b64_png() -> str: - import base64 - return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode() + return base64.b64encode(_png_bytes()).decode() @pytest.fixture(autouse=True) def _tmp_hermes_home(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.delenv("OPENAI_IMAGE_MODEL", raising=False) yield tmp_path @@ -46,6 +51,33 @@ def provider(monkeypatch): return codex_plugin.OpenAICodexImageGenProvider() +@pytest.fixture +def codex_backend(monkeypatch): + """Route the plugin's ``httpx.Client`` at a fake Codex images backend; returns the request log + and lets a test swap the response via ``state["respond"]``.""" + monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") + state = {"requests": [], "respond": None} + + def _default(request): + return httpx.Response(200, json={ + "created": 1, "data": [{"b64_json": _b64_png(), "generation_id": "gen_1"}], + "background": "opaque", "output_format": "png", "quality": "low", "size": "1254x1254", + }, headers={"x-codex-imagegen-request-id": "req_abc"}, request=request) + + def _handler(request): + state["requests"].append(request) + return (state["respond"] or _default)(request) + + real_client = httpx.Client + monkeypatch.setattr( + httpx, "Client", + lambda *args, **kwargs: real_client( + transport=httpx.MockTransport(_handler), headers=kwargs.get("headers"), + timeout=kwargs.get("timeout")), + ) + return state + + # ── Metadata ──────────────────────────────────────────────────────────────── @@ -66,7 +98,7 @@ def test_list_models_three_tiers(self, provider): def test_setup_schema_has_no_required_env_vars(self, provider): schema = provider.get_setup_schema() assert schema["env_vars"] == [] - assert schema["badge"] == "free" + assert "hermes auth codex" in schema["post_setup_hint"] # ── Availability ──────────────────────────────────────────────────────────── @@ -74,24 +106,20 @@ def test_setup_schema_has_no_required_env_vars(self, provider): class TestAvailability: def test_unavailable_without_codex_token(self, monkeypatch): - monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None) assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False def test_available_with_codex_token(self, monkeypatch): - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") + monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "tok") assert codex_plugin.OpenAICodexImageGenProvider().is_available() is True def test_openai_api_key_alone_is_not_enough(self, monkeypatch): - # Codex plugin is intentionally orthogonal to the API-key plugin — - # the API key alone must NOT make it appear available. monkeypatch.setenv("OPENAI_API_KEY", "sk-test") monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: None) assert codex_plugin.OpenAICodexImageGenProvider().is_available() is False -# ── Generate ──────────────────────────────────────────────────────────────── +# ── Generation ────────────────────────────────────────────────────────────── class TestGenerate: @@ -101,354 +129,100 @@ def test_returns_auth_error_without_codex_token(self, provider, monkeypatch): assert result["success"] is False assert result["error_type"] == "auth_required" - - def test_generate_uses_codex_stream_path(self, provider, monkeypatch, tmp_path): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: {"b64": _b64_png(), "source": "final"}) - - result = provider.generate("a cat", aspect_ratio="landscape") + def test_text_to_image_posts_generations_with_no_host_model(self, provider, codex_backend, tmp_path): + result = provider.generate("a cat", aspect_ratio="portrait") assert result["success"] is True assert result["model"] == "gpt-image-2-medium" assert result["provider"] == "openai-codex" assert result["quality"] == "medium" - assert result.get("image_source") == "final" - assert result.get("pixel_size") == "1x1" - + assert result["pixel_size"] == "1x1" + # Backend-reported values travel separately from what we asked for (#107233). + assert result["reported_quality"] == "low" + assert result["reported_size"] == "1254x1254" + assert result["imagegen_request_id"] == "req_abc" saved = Path(result["image"]) - assert saved.exists() - assert saved.parent == tmp_path / "cache" / "images" - # Filename prefix differs from the API-key plugin so cache audits can - # tell the two backends apart. + assert saved.exists() and saved.parent == tmp_path / "cache" / "images" assert saved.name.startswith("openai_codex_") - def test_codex_stream_request_shape(self, provider, monkeypatch): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - - captured = {} + (request,) = codex_backend["requests"] + assert request.url.path.endswith("/backend-api/codex/images/generations") + assert request.headers["Authorization"] == "Bearer codex-token" + assert request.headers["x-codex-image-turn-id"] + body = json.loads(request.content) + assert body == { + "prompt": "a cat", "model": "gpt-image-2", "n": 1, "quality": "medium", + "size": "1024x1536", "background": "opaque", + } + # The whole point of the native route: nothing about a chat model in the request. + assert not any(key in body for key in ("tools", "input", "instructions")) - def _collect(token, *, prompt, size, quality, input_images=None): - captured.update(codex_plugin._build_responses_payload( - prompt=prompt, - size=size, - quality=quality, - input_images=input_images, - )) - return {"b64": _b64_png(), "source": "final"} + def test_source_images_post_edits_with_inline_data_urls(self, provider, codex_backend, tmp_path): + local = tmp_path / "ref.png" + local.write_bytes(_png_bytes()) + data_url = "data:image/png;base64," + _b64_png() - monkeypatch.setattr(codex_plugin, "_collect_image_b64", _collect) + result = provider.generate("edit these", image_url=str(local), reference_image_urls=[data_url]) - result = provider.generate("a cat", aspect_ratio="portrait") assert result["success"] is True + assert result["modality"] == "image" + assert result["input_image_count"] == 2 + (request,) = codex_backend["requests"] + assert request.url.path.endswith("/backend-api/codex/images/edits") + body = json.loads(request.content) + assert [img["image_url"] for img in body["images"]] == [data_url, data_url] + + def test_remote_source_url_is_fetched_and_inlined(self, provider, codex_backend, monkeypatch): + # The backend's own URL downloader 400s on ordinary public images; we fetch client-side. + monkeypatch.setattr( + httpx, "get", + lambda url, **kw: httpx.Response(200, content=_png_bytes(), request=httpx.Request("GET", url))) - assert captured["model"] == "gpt-5.5" - assert captured["store"] is False - assert captured["input"][0]["type"] == "message" - assert captured["input"][0]["role"] == "user" - assert captured["input"][0]["content"][0]["type"] == "input_text" - # Regression for #19505: the Codex backend 400s on every tool_choice - # shape we have for the hosted ``image_generation`` tool, so the - # provider must omit tool_choice entirely and rely on instructions. - assert "tool_choice" not in captured - - tool = captured["tools"][0] - assert tool["type"] == "image_generation" - assert tool["model"] == "gpt-image-2" - assert tool["quality"] == "medium" - assert tool["size"] == "1024x1536" - assert tool["output_format"] == "png" - assert tool["background"] == "opaque" - # Progressive previews disabled: partial frames were being saved as - # finals and presented as smeared/unfinished images. - assert tool["partial_images"] == 0 + result = provider.generate("edit", image_url="https://example.com/ref.png") + + assert result["success"] is True + body = json.loads(codex_backend["requests"][0].content) + assert body["images"] == [{"image_url": "data:image/png;base64," + _b64_png()}] def test_capabilities_advertise_image_inputs(self, provider): caps = provider.capabilities() assert caps["modalities"] == ["text", "image"] assert caps["max_reference_images"] == 16 - - def test_rejects_non_image_local_source(self, provider, monkeypatch, tmp_path): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") + def test_rejects_non_image_local_source(self, provider, codex_backend, tmp_path): text_path = tmp_path / "not-image.txt" - text_path.write_text("hello") + text_path.write_text("hello", encoding="utf-8") result = provider.generate("edit this", image_url=str(text_path)) assert result["success"] is False assert result["error_type"] == "invalid_image_input" assert "not a supported image" in result["error"] + assert codex_backend["requests"] == [] - - def test_partial_image_event_used_when_done_missing(self): - """Extractor may surface partial b64 when no final exists (fallback only).""" - payload = { - "type": "response.image_generation_call.partial_image", - "partial_image_b64": _b64_png(), - } - assert codex_plugin._extract_image_b64(payload) == _b64_png() - result, partial = codex_plugin._extract_image_candidates(payload) - assert result is None - assert partial == _b64_png() - - def test_final_result_wins_over_coexisting_partial_in_same_payload(self): - """Blind spot that shipped the smear bug: both fields in one payload. - - partial_image_b64 must never overwrite image_generation_call.result - when they coexist in the same event tree. - """ - final = _b64_png() - # Distinct non-empty stand-in so equality proves which field won. - partial = "cGFydGlhbC1vbmx5LW5vdC1hLXJlYWwtZmluYWw=" - payload = { - "type": "response.output_item.done", - "item": { - "type": "image_generation_call", - "status": "completed", - "result": final, - "partial_image_b64": partial, - }, - } - assert codex_plugin._extract_image_b64(payload) == final - result, got_partial = codex_plugin._extract_image_candidates(payload) - assert result == final - assert got_partial == partial - - def test_nested_final_wins_over_sibling_partial(self): - payload = { - "type": "response.completed", - "response": { - "output": [{ - "type": "image_generation_call", - "status": "completed", - "result": _b64_png(), - }], - }, - "partial_image_b64": "cGFydGlhbC1zaWJsaW5n", - } - assert codex_plugin._extract_image_b64(payload) == _b64_png() - - def test_sse_parser_handles_event_and_data_lines(self): - class _Response: - def iter_lines(self): - return iter([ - "event: response.output_item.done", - 'data: {"item": {"type": "image_generation_call", "result": "abc"}}', - "", - ]) - - events = list(codex_plugin._iter_sse_json(_Response())) - assert events == [{ - "type": "response.output_item.done", - "item": {"type": "image_generation_call", "result": "abc"}, - }] - - def test_final_response_sweep_recovers_image(self): - """Completed response output is found by recursive payload scanning.""" - payload = { - "type": "response.completed", - "response": { - "output": [{ - "type": "image_generation_call", - "status": "completed", - "id": "ig_final", - "result": _b64_png(), - }], - }, - } - assert codex_plugin._extract_image_b64(payload) == _b64_png() - - def test_partial_only_stream_fails_closed_after_retry(self, provider, monkeypatch): - """Partial-only streams must not return success:true with a smear frame.""" - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - calls = {"n": 0} - - def _partial_only(*args, **kwargs): - calls["n"] += 1 - return {"b64": _b64_png(), "source": "partial"} - - monkeypatch.setattr(codex_plugin, "_collect_image_b64", _partial_only) - - result = provider.generate("a cat") - assert result["success"] is False - assert result["error_type"] == "incomplete_image" - assert "partial" in result["error"].lower() - # One initial attempt + one content-agnostic retry. - assert calls["n"] == codex_plugin._NONFINAL_RETRIES + 1 - - def test_empty_stream_retries_then_fails(self, provider, monkeypatch): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - calls = {"n": 0} - - def _empty(*args, **kwargs): - calls["n"] += 1 - return None - - monkeypatch.setattr(codex_plugin, "_collect_image_b64", _empty) - - result = provider.generate("a cat") - assert result["success"] is False - assert result["error_type"] == "empty_response" - assert calls["n"] == codex_plugin._NONFINAL_RETRIES + 1 - - def test_partial_then_final_on_retry_succeeds(self, provider, monkeypatch): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - calls = {"n": 0} - - def _then_final(*args, **kwargs): - calls["n"] += 1 - if calls["n"] == 1: - return {"b64": _b64_png(), "source": "partial"} - return {"b64": _b64_png(), "source": "final"} - - monkeypatch.setattr(codex_plugin, "_collect_image_b64", _then_final) - - result = provider.generate("a cat") - assert result["success"] is True - assert result.get("image_source") == "final" - assert calls["n"] == 2 - - def test_empty_then_final_on_retry_succeeds(self, provider, monkeypatch): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - calls = {"n": 0} - - def _then_final(*args, **kwargs): - calls["n"] += 1 - if calls["n"] == 1: - return None - return {"b64": _b64_png(), "source": "final"} - - monkeypatch.setattr(codex_plugin, "_collect_image_b64", _then_final) - - result = provider.generate("a cat") - assert result["success"] is True - assert result.get("image_source") == "final" - assert calls["n"] == 2 - - def test_empty_response_returns_error(self, provider, monkeypatch): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - monkeypatch.setattr(codex_plugin, "_NONFINAL_RETRIES", 0) - monkeypatch.setattr(codex_plugin, "_collect_image_b64", lambda *a, **kw: None) - - result = provider.generate("a cat") - assert result["success"] is False - assert result["error_type"] == "empty_response" - - def test_stream_exception_returns_api_error(self, provider, monkeypatch): - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - - def _boom(*args, **kwargs): - raise RuntimeError("cloudflare 403") - - monkeypatch.setattr(codex_plugin, "_collect_image_b64", _boom) - - result = provider.generate("a cat") - assert result["success"] is False - assert result["error_type"] == "api_error" - assert "cloudflare 403" in result["error"] - - def test_tool_choice_400_surfaces_verbatim_not_as_capability_error( - self, provider, monkeypatch - ): - """The tool_choice 400 must NOT be reported as an account limitation. - - Regression for #19505 / #49008 / #31335: a previous version classified - this exact request-shape rejection as "Image generation is not enabled - for the current Codex account", telling every affected user to abandon - Codex over a bug in our own payload. The wire error must reach the user - unedited so it stays diagnosable. - - Drives the REAL httpx boundary (not a mocked ``_collect_image_b64``) so - the classification path is actually exercised — mocking the collector - would skip the code under test entirely. - """ - import httpx - - monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token") - + def test_http_error_message_surfaces_verbatim_and_bounded(self, provider, codex_backend): body = json.dumps({ - "error": { - "message": "Tool choice 'image_generation' not found in 'tools' parameter.", - "type": "invalid_request_error", - "param": "tool_choice", - } + "metadata": "x" * 600, + "error": {"message": "Missing required parameter: 'prompt'.", "type": "invalid_request_error"}, }) - - def _handler(request): - return httpx.Response(400, text=body, request=request) - - real_client = httpx.Client - monkeypatch.setattr( - httpx, - "Client", - lambda *args, **kwargs: real_client( - transport=httpx.MockTransport(_handler), - headers=kwargs.get("headers"), - timeout=kwargs.get("timeout"), - ), - ) + codex_backend["respond"] = lambda request: httpx.Response(400, text=body, request=request) result = provider.generate("a cat") assert result["success"] is False assert result["error_type"] == "api_error" assert "HTTP 400" in result["error"] - assert "tools' parameter" in result["error"] - # The account-entitlement misdiagnosis must not come back. - assert "not enabled for the current Codex account" not in result["error"] - assert result["error_type"] != "capability_unsupported" - - -class TestRequestShape: - def test_payload_omits_tool_choice(self): - """Codex rejects every tool_choice shape for hosted image_generation.""" - payload = codex_plugin._build_responses_payload( - prompt="a red circle", - size="1024x1024", - quality="low", - ) - assert "tool_choice" not in payload - # The hosted tool itself is still requested, and instructions do the steering. - assert payload["tools"][0]["type"] == "image_generation" - assert payload["instructions"] - - def test_http_error_body_is_truncated_but_preserved(self, monkeypatch): - """A large error body is capped at 500 chars and still surfaced.""" - import httpx + assert "Missing required parameter: 'prompt'." in result["error"] + assert len(result["error"]) < len(body) - body = json.dumps({ - "metadata": "x" * 600, - "error": { - "message": "Tool choice 'image_generation' not found in 'tools' parameter." - }, - }) + def test_missing_image_data_is_empty_response(self, provider, codex_backend): + codex_backend["respond"] = lambda request: httpx.Response( + 200, json={"created": 1, "data": []}, request=request) - def _handler(request): - return httpx.Response(400, text=body, request=request) + result = provider.generate("a cat") - real_client = httpx.Client - monkeypatch.setattr( - httpx, - "Client", - lambda *args, **kwargs: real_client( - transport=httpx.MockTransport(_handler), - headers=kwargs.get("headers"), - timeout=kwargs.get("timeout"), - ), - ) - - with pytest.raises(RuntimeError, match="HTTP 400") as excinfo: - codex_plugin._collect_image_b64( - "codex-token", - prompt="a cat", - size="1024x1024", - quality="low", - ) - - message = str(excinfo.value) - # Body is capped, but the actionable wire message still reaches the user. - assert "tools' parameter" in message - assert len(message) < len(body) + assert result["success"] is False + assert result["error_type"] == "empty_response" # ── Plugin entry point ────────────────────────────────────────────────────── diff --git a/website/docs/developer-guide/image-gen-provider-plugin.md b/website/docs/developer-guide/image-gen-provider-plugin.md index a42aa3c97459b..ea4dc7051607a 100644 --- a/website/docs/developer-guide/image-gen-provider-plugin.md +++ b/website/docs/developer-guide/image-gen-provider-plugin.md @@ -297,7 +297,7 @@ Or interactively: `hermes tools` → "Image Generation" → select `my-backend` - **`plugins/image_gen/openai/__init__.py`** — gpt-image-2 at low/medium/high tiers as three virtual model IDs sharing one API model with different `quality` params. Good example of tiered models under a single backend + config.yaml precedence chain. - **`plugins/image_gen/xai/__init__.py`** — Grok Imagine via xAI. Different shape (URL output, simpler catalog). -- **`plugins/image_gen/openai-codex/__init__.py`** — Codex-style Responses API variant reusing the OpenAI SDK with a different routing base URL. +- **`plugins/image_gen/openai-codex/__init__.py`** — same catalog as `openai`, but authenticated with the ChatGPT/Codex OAuth token and posted with plain `httpx` to the Codex backend's native `images/generations` / `images/edits` endpoints. Good example of a provider that fetches remote source images client-side and inlines them as data URLs, and that reports backend-returned metadata separately from the request. ## Distribute via pip diff --git a/website/docs/user-guide/features/image-generation.md b/website/docs/user-guide/features/image-generation.md index 862ee885d19f8..c5a753eb42511 100644 --- a/website/docs/user-guide/features/image-generation.md +++ b/website/docs/user-guide/features/image-generation.md @@ -182,10 +182,10 @@ does not estimate 2.5 token consumption. See the official [Flare](https://developers.openai.com/api/docs/models/gpt-image-2.5-flare) and [Sunburst](https://developers.openai.com/api/docs/models/gpt-image-2.5-sunburst) docs. -The **OpenAI (Codex auth)** provider remains separate: its backend can accept -an image-model value without honoring that selection, so a successful image -alone does not verify Flare or Sunburst routing. These selections are offered -through the direct OpenAI API provider and FAL, not as verified Codex-auth selections. +The **OpenAI (Codex auth)** provider does not offer 2.5. The Codex backend +accepts any `model` value (including nonexistent ids) and generates with its +own server-managed engine, so a "selected" Flare or Sunburst tier would be a +label with no effect. Pick the direct OpenAI API provider or FAL for 2.5. ## Usage @@ -231,7 +231,7 @@ Two inputs drive the edit: | **OpenAI** (GPT Image 2 / 2.5 Flare / Sunburst) | ✓ | up to 16 | `images.edit()` | | **xAI** (Grok Imagine) | ✓ | 1 | `/v1/images/edits` (`grok-imagine-image-quality`) | | **Krea** (`Krea 2`) | ✓ | up to 10 | reference-guided generation (`image_style_references`) | -| **OpenAI (Codex auth)** | ✓ | up to 16 | Codex Responses `image_generation` tool with `input_image` content parts | +| **OpenAI (Codex auth)** | ✓ | up to 16 | `POST /backend-api/codex/images/edits` with inline `images[]` data URLs (remote URLs are fetched client-side) | | **OpenRouter** (Image API models) | ✓ | up to 14–16 (per model) | `input_references` on `POST /images/generations`; chat-served models use `image_url` content parts (up to 3) | FAL models with an editing endpoint: `flux-2/klein/9b`, `flux-2-pro`, @@ -240,16 +240,18 @@ FAL models with an editing endpoint: `flux-2/klein/9b`, `flux-2-pro`, `krea/*`) reject image inputs with a clear error pointing you at an edit-capable model. -:::note OpenAI (Codex auth) is best-effort - -The Codex surface (`chatgpt.com/backend-api/codex`) hosts `image_generation` -as a tool the chat model may call, and Hermes cannot force the call — the -backend rejects every `tool_choice` shape for hosted tools, so the request -relies on instructions to steer the model. When the host model declines to -invoke the tool, the call fails with `empty_response`. Whether the hosted -image tool is reachable at all has also been reported to vary between -accounts. If you need image generation to work deterministically, configure -the **OpenAI** (API key), **FAL**, or **xAI** backend instead. +:::note OpenAI (Codex auth): the backend decides quality and size + +Hermes posts straight to the Codex backend's native +`images/generations` / `images/edits` endpoints (the same route the official +Codex client uses), so no chat model is involved and the call does not depend +on which chat models your ChatGPT plan currently has. The backend, however, +treats `model`, `quality` and `size` as advisory: it may return a different +quality tier or geometry than requested (a portrait request can come back +square). The result carries `reported_quality`, `reported_size` and +`pixel_size` alongside what was requested, plus `imagegen_request_id` for +OpenAI support. For exact control over quality and size, configure the +**OpenAI** (API key), **FAL**, or **xAI** backend instead. :::