feat(image_gen): multi-reference input support, starting with openai-codex

agent/image_gen_provider.py

@@ -76,6 +76,18 @@ def is_available(self) -> bool:
         """
         return True
 
+    @property
+    def supports_references(self) -> bool:
+        """True when :meth:`generate` accepts a ``references`` kwarg.
+
+        When True, the ``image_generate`` tool schema exposes a ``references``
+        field and the dispatcher forwards user-supplied image paths to this
+        provider. When False (the default), the dispatcher rejects calls that
+        include references with a clear ``references_unsupported`` error
+        instead of silently dropping them.
+        """
+        return False
+
     def list_models(self) -> List[Dict[str, Any]]:
         """Return catalog entries for ``hermes tools`` model picker.
 
@@ -140,6 +152,13 @@ def generate(
         or :func:`error_response`. ``kwargs`` may contain forward-compat
         parameters future versions of the schema will expose — implementations
         should ignore unknown keys.
+
+        Known optional kwargs (implementations opt-in by overriding the
+        matching capability property):
+
+        - ``references``: ``list[str]`` of local image file paths, forwarded
+          only when :attr:`supports_references` is True. Typically used for
+          image-to-image editing or multi-reference composition.
         """
 
 

plugins/image_gen/openai-codex/__init__.py

@@ -7,6 +7,13 @@
 authenticated with Codex/ChatGPT generate images without configuring a
 separate ``OPENAI_API_KEY``.
 
+Also supports **multi-reference input** (up to 16 images): callers pass
+``references=[path1, path2, ...]`` to :meth:`OpenAICodexImageGenProvider.generate`
+and each file becomes an ``input_image`` content item on the user message. The
+Codex ``image_generation`` tool sees them and uses them for style transfer,
+compositing, or image-to-image editing — verified empirically against the
+live backend.
+
 Selection precedence for the tier (first hit wins):
 
 1. ``OPENAI_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
@@ -19,8 +26,11 @@
 
 from __future__ import annotations
 
+import base64
 import logging
-from typing import Any, Dict, List, Optional, Tuple
+import mimetypes
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Tuple
 
 from agent.image_gen_provider import (
     DEFAULT_ASPECT_RATIO,
@@ -63,6 +73,10 @@
 
 DEFAULT_MODEL = "gpt-image-2-medium"
 
+# Upper bound on the number of reference images accepted in one call —
+# matches OpenAI's documented cap for gpt-image-2 edit/composition flows.
+MAX_REFERENCES = 16
+
 _SIZES = {
     "landscape": "1536x1024",
     "square": "1024x1024",
@@ -161,7 +175,70 @@ def _build_codex_client():
         return None
 
 
-def _collect_image_b64(client: Any, *, prompt: str, size: str, quality: str) -> Optional[str]:
+def _load_reference_images(
+    paths: Sequence[Any],
+) -> Tuple[List[Tuple[str, str]], Optional[str]]:
+    """Load reference images from disk into (mime, base64) pairs.
+
+    Returns ``(loaded, error)``. ``loaded`` is capped at :data:`MAX_REFERENCES`.
+    ``error`` is a short string describing the first invalid entry, or None
+    when every path resolved to a readable image file.
+    """
+    loaded: List[Tuple[str, str]] = []
+    for raw in paths:
+        if len(loaded) >= MAX_REFERENCES:
+            break
+        if not isinstance(raw, (str, Path)):
+            return loaded, f"reference path must be a string, got {type(raw).__name__}"
+        path = Path(str(raw)).expanduser()
+        if not path.is_file():
+            return loaded, f"reference not found: {path}"
+        mime, _ = mimetypes.guess_type(path.name)
+        if not mime or not mime.startswith("image/"):
+            mime = "image/png"
+        try:
+            encoded = base64.b64encode(path.read_bytes()).decode()
+        except OSError as exc:
+            return loaded, f"could not read reference {path}: {exc}"
+        loaded.append((mime, encoded))
+    return loaded, None
+
+
+def _build_user_content(
+    prompt: str,
+    references: Sequence[Tuple[str, str]],
+) -> List[Dict[str, Any]]:
+    """Build the Responses API ``content`` array for the user message.
+
+    Multiple references are labelled in the prompt ("Reference image 1 …")
+    so the user's instructions can refer to them by index — matches OpenAI's
+    published best practice for multi-reference composition.
+    """
+    if references:
+        labelled = prompt + "\n\n" + "\n".join(
+            f"Reference image {i + 1} is provided below."
+            for i in range(len(references))
+        )
+    else:
+        labelled = prompt
+
+    content: List[Dict[str, Any]] = [{"type": "input_text", "text": labelled}]
+    for mime, b64 in references:
+        content.append({
+            "type": "input_image",
+            "image_url": f"data:{mime};base64,{b64}",
+        })
+    return content
+
+
+def _collect_image_b64(
+    client: Any,
+    *,
+    prompt: str,
+    size: str,
+    quality: str,
+    references: Sequence[Tuple[str, str]] = (),
+) -> Optional[str]:
     """Stream a Codex Responses image_generation call and return the b64 image."""
     image_b64: Optional[str] = None
 
@@ -172,7 +249,7 @@ def _collect_image_b64(client: Any, *, prompt: str, size: str, quality: str) ->
         input=[{
             "type": "message",
             "role": "user",
-            "content": [{"type": "input_text", "text": prompt}],
+            "content": _build_user_content(prompt, references),
         }],
         tools=[{
             "type": "image_generation",
@@ -230,6 +307,13 @@ def name(self) -> str:
     def display_name(self) -> str:
         return "OpenAI (Codex auth)"
 
+    @property
+    def supports_references(self) -> bool:
+        # The Codex ``image_generation`` tool accepts ``input_image`` content
+        # items on the user message and uses them for composition, style
+        # transfer, and edits — verified empirically against the live backend.
+        return True
+
     def is_available(self) -> bool:
         if not _read_codex_access_token():
             return False
@@ -307,6 +391,31 @@ def generate(
         tier_id, meta = _resolve_model()
         size = _SIZES.get(aspect, _SIZES["square"])
 
+        raw_refs = kwargs.get("references") or []
+        if not isinstance(raw_refs, (list, tuple)):
+            return error_response(
+                error=(
+                    f"references must be a list of image paths, got "
+                    f"{type(raw_refs).__name__}"
+                ),
+                error_type="invalid_argument",
+                provider="openai-codex",
+                model=tier_id,
+                prompt=prompt,
+                aspect_ratio=aspect,
+            )
+
+        references, ref_error = _load_reference_images(raw_refs)
+        if ref_error is not None:
+            return error_response(
+                error=ref_error,
+                error_type="invalid_reference",
+                provider="openai-codex",
+                model=tier_id,
+                prompt=prompt,
+                aspect_ratio=aspect,
+            )
+
         client = _build_codex_client()
         if client is None:
             return error_response(
@@ -324,6 +433,7 @@ def generate(
                 prompt=prompt,
                 size=size,
                 quality=meta["quality"],
+                references=references,
             )
         except Exception as exc:
             logger.debug("Codex image generation failed", exc_info=True)
@@ -364,7 +474,11 @@ def generate(
             prompt=prompt,
             aspect_ratio=aspect,
             provider="openai-codex",
-            extra={"size": size, "quality": meta["quality"]},
+            extra={
+                "size": size,
+                "quality": meta["quality"],
+                "references": len(references),
+            },
         )
 
 

tests/plugins/image_gen/test_openai_codex_provider.py

@@ -283,6 +283,158 @@ def _boom(**kwargs):
         assert "cloudflare 403" in result["error"]
 
 
+# ── References (multi-reference image input) ───────────────────────────────
+
+
+def _write_png(path: Path) -> Path:
+    path.write_bytes(bytes.fromhex(_PNG_HEX))
+    return path
+
+
+class TestReferences:
+    def test_supports_references_flag_is_true(self, provider):
+        # The Codex image_generation tool accepts input_image content items;
+        # the flag is how the dispatcher knows it can forward user-supplied
+        # reference paths without silently dropping them.
+        assert provider.supports_references is True
+
+    def test_references_become_input_image_content_items(
+        self, provider, monkeypatch, tmp_path
+    ):
+        monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
+
+        ref1 = _write_png(tmp_path / "ref1.png")
+        ref2 = _write_png(tmp_path / "ref2.png")
+
+        captured: dict = {}
+
+        def _stream(**kwargs):
+            captured.update(kwargs)
+            output_item = SimpleNamespace(
+                type="image_generation_call",
+                status="generating",
+                id="ig_test",
+                result=_b64_png(),
+            )
+            done_event = SimpleNamespace(type="response.output_item.done", item=output_item)
+            final_response = SimpleNamespace(output=[], status="completed", output_text="")
+            return _FakeStream([done_event], final_response)
+
+        fake_client = SimpleNamespace(responses=SimpleNamespace(stream=_stream))
+        monkeypatch.setattr(codex_plugin, "_build_codex_client", lambda: fake_client)
+
+        result = provider.generate(
+            "combine these two objects",
+            aspect_ratio="square",
+            references=[str(ref1), str(ref2)],
+        )
+        assert result["success"] is True
+        assert result["references"] == 2
+
+        content = captured["input"][0]["content"]
+        # First item is the labelled prompt, then one input_image per reference.
+        assert content[0]["type"] == "input_text"
+        assert "Reference image 1" in content[0]["text"]
+        assert "Reference image 2" in content[0]["text"]
+        image_items = [c for c in content if c["type"] == "input_image"]
+        assert len(image_items) == 2
+        for item in image_items:
+            assert item["image_url"].startswith("data:image/")
+            assert ";base64," in item["image_url"]
+
+    def test_references_cap_at_max(self, provider, monkeypatch, tmp_path):
+        monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
+
+        # 20 valid PNG paths — the plugin should only forward the first 16.
+        paths = [str(_write_png(tmp_path / f"ref{i}.png")) for i in range(20)]
+
+        captured: dict = {}
+
+        def _stream(**kwargs):
+            captured.update(kwargs)
+            output_item = SimpleNamespace(
+                type="image_generation_call",
+                status="generating",
+                id="ig_test",
+                result=_b64_png(),
+            )
+            done_event = SimpleNamespace(type="response.output_item.done", item=output_item)
+            final_response = SimpleNamespace(output=[], status="completed", output_text="")
+            return _FakeStream([done_event], final_response)
+
+        fake_client = SimpleNamespace(responses=SimpleNamespace(stream=_stream))
+        monkeypatch.setattr(codex_plugin, "_build_codex_client", lambda: fake_client)
+
+        result = provider.generate("test", references=paths)
+        assert result["success"] is True
+        assert result["references"] == codex_plugin.MAX_REFERENCES
+
+        image_items = [
+            c for c in captured["input"][0]["content"] if c["type"] == "input_image"
+        ]
+        assert len(image_items) == codex_plugin.MAX_REFERENCES
+
+    def test_missing_reference_returns_invalid_reference(
+        self, provider, monkeypatch, tmp_path
+    ):
+        monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
+        monkeypatch.setattr(
+            codex_plugin,
+            "_build_codex_client",
+            lambda: SimpleNamespace(
+                responses=SimpleNamespace(
+                    stream=lambda **kw: pytest.fail("stream must not be called on invalid ref")
+                )
+            ),
+        )
+
+        result = provider.generate(
+            "test",
+            references=[str(tmp_path / "does-not-exist.png")],
+        )
+        assert result["success"] is False
+        assert result["error_type"] == "invalid_reference"
+        assert "does-not-exist.png" in result["error"]
+
+    def test_non_list_references_rejected(self, provider, monkeypatch):
+        monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
+
+        result = provider.generate("test", references="not-a-list")
+        assert result["success"] is False
+        assert result["error_type"] == "invalid_argument"
+
+    def test_zero_references_behaves_like_prompt_only(
+        self, provider, monkeypatch
+    ):
+        # Regression guard: an explicit empty list must not add Reference-image
+        # labelling to the prompt or any input_image items.
+        monkeypatch.setattr(codex_plugin, "_read_codex_access_token", lambda: "codex-token")
+
+        captured: dict = {}
+
+        def _stream(**kwargs):
+            captured.update(kwargs)
+            output_item = SimpleNamespace(
+                type="image_generation_call",
+                status="generating",
+                id="ig_test",
+                result=_b64_png(),
+            )
+            done_event = SimpleNamespace(type="response.output_item.done", item=output_item)
+            final_response = SimpleNamespace(output=[], status="completed", output_text="")
+            return _FakeStream([done_event], final_response)
+
+        fake_client = SimpleNamespace(responses=SimpleNamespace(stream=_stream))
+        monkeypatch.setattr(codex_plugin, "_build_codex_client", lambda: fake_client)
+
+        result = provider.generate("a cat", references=[])
+        assert result["success"] is True
+        assert result["references"] == 0
+        content = captured["input"][0]["content"]
+        assert content[0]["text"] == "a cat"
+        assert all(c["type"] != "input_image" for c in content)
+
+
 # ── Plugin entry point ──────────────────────────────────────────────────────