From 6502312d79a4381919443393f0bc634dc8a44c74 Mon Sep 17 00:00:00 2001 From: y0shualee Date: Sat, 2 May 2026 19:04:30 +0800 Subject: [PATCH] fix(plugins): support OpenAI image API model overrides Split the OpenAI image plugin's virtual quality-tier IDs from the actual API model sent in images.generate requests. This keeps gpt-image-2-low/medium/high as UI/config tier selectors while allowing OpenAI-compatible backends to set the real payload model via OPENAI_IMAGE_API_MODEL, image_gen.openai.api_model, or the dispatcher-provided image_gen.model from #21273 when it is not one of the virtual tier IDs. Also pass OPENAI_BASE_URL or image_gen.openai.base_url into openai.OpenAI(), and cover the resolver, provider.generate, and dispatcher model forwarding paths with tests. Closes: NousResearch/hermes-agent#18793 --- plugins/image_gen/openai/__init__.py | 140 ++++++++++++++-- .../plugins/image_gen/test_openai_provider.py | 152 ++++++++++++++++++ .../test_image_generation_plugin_dispatch.py | 48 ++++++ 3 files changed, 326 insertions(+), 14 deletions(-) diff --git a/plugins/image_gen/openai/__init__.py b/plugins/image_gen/openai/__init__.py index cfa9e42c908fe..3fd8ec4b5e791 100644 --- a/plugins/image_gen/openai/__init__.py +++ b/plugins/image_gen/openai/__init__.py @@ -13,12 +13,27 @@ different ``quality`` parameter. Output is base64 JSON → saved under ``$HERMES_HOME/cache/images/``. -Selection precedence (first hit wins): +Tier selection precedence (first hit wins): -1. ``OPENAI_IMAGE_MODEL`` env var (escape hatch for scripts / tests) +1. ``OPENAI_IMAGE_MODEL`` env var 2. ``image_gen.openai.model`` in ``config.yaml`` 3. ``image_gen.model`` in ``config.yaml`` (when it's one of our tier IDs) 4. :data:`DEFAULT_MODEL` — ``gpt-image-2-medium`` + +API model selection precedence (the actual ``model`` field in the API call): + +1. ``OPENAI_IMAGE_API_MODEL`` env var — escape hatch for third-party backends +2. ``image_gen.openai.api_model`` in ``config.yaml`` +3. ``model`` passed to :meth:`generate` (from ``image_gen.model`` in the dispatcher) + when it is not one of this plugin's virtual quality-tier IDs +4. ``image_gen.model`` in ``config.yaml`` when it is not a virtual tier ID +5. ``API_MODEL`` — ``gpt-image-2`` + +Base URL selection precedence: + +1. ``OPENAI_BASE_URL`` env var +2. ``image_gen.openai.base_url`` in ``config.yaml`` +3. ``None`` — defaults to api.openai.com """ from __future__ import annotations @@ -94,9 +109,24 @@ def _load_openai_config() -> Dict[str, Any]: return {} -def _resolve_model() -> Tuple[str, Dict[str, Any]]: - """Decide which tier to use and return ``(model_id, meta)``.""" - env_override = os.environ.get("OPENAI_IMAGE_MODEL") +def _non_empty(value: Any) -> Optional[str]: + """Return a stripped string when ``value`` is non-empty, else ``None``.""" + if isinstance(value, str): + stripped = value.strip() + if stripped: + return stripped + return None + + +def _resolve_model(requested_model: Optional[str] = None) -> Tuple[str, Dict[str, Any]]: + """Decide which quality tier to use and return ``(model_id, meta)``. + + ``requested_model`` is the ``model`` kwarg passed by the tool dispatcher + (currently sourced from top-level ``image_gen.model``). It only affects + tier selection when it is one of this plugin's virtual tier IDs; other + values are treated as actual API model names by ``_resolve_api_model()``. + """ + env_override = _non_empty(os.environ.get("OPENAI_IMAGE_MODEL")) if env_override and env_override in _MODELS: return env_override, _MODELS[env_override] @@ -104,12 +134,16 @@ def _resolve_model() -> Tuple[str, Dict[str, Any]]: openai_cfg = cfg.get("openai") if isinstance(cfg.get("openai"), dict) else {} candidate: Optional[str] = None if isinstance(openai_cfg, dict): - value = openai_cfg.get("model") - if isinstance(value, str) and value in _MODELS: + value = _non_empty(openai_cfg.get("model")) + if value in _MODELS: candidate = value if candidate is None: - top = cfg.get("model") - if isinstance(top, str) and top in _MODELS: + requested = _non_empty(requested_model) + if requested in _MODELS: + candidate = requested + if candidate is None: + top = _non_empty(cfg.get("model")) + if top in _MODELS: candidate = top if candidate is not None: @@ -118,6 +152,67 @@ def _resolve_model() -> Tuple[str, Dict[str, Any]]: return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL] +def _resolve_api_model(requested_model: Optional[str] = None) -> str: + """Resolve the actual model name to send in the API request. + + Selection precedence (first hit wins): + + 1. ``OPENAI_IMAGE_API_MODEL`` env var — escape hatch for third-party backends + 2. ``image_gen.openai.api_model`` in ``config.yaml`` + 3. ``requested_model`` when it is not a virtual quality-tier ID + 4. ``image_gen.model`` in ``config.yaml`` when it is not a virtual tier ID + 5. ``API_MODEL`` — ``gpt-image-2`` + + This is separate from ``_resolve_model()`` because the tier selection + (low/medium/high) and the API model name are orthogonal concerns. + """ + env_override = _non_empty(os.environ.get("OPENAI_IMAGE_API_MODEL")) + if env_override: + return env_override + + cfg = _load_openai_config() + openai_cfg = cfg.get("openai") if isinstance(cfg.get("openai"), dict) else {} + if isinstance(openai_cfg, dict): + value = _non_empty(openai_cfg.get("api_model")) + if value: + return value + + requested = _non_empty(requested_model) + if requested and requested not in _MODELS: + return requested + + top = _non_empty(cfg.get("model")) + if top and top not in _MODELS: + return top + + return API_MODEL + + +def _resolve_base_url() -> Optional[str]: + """Resolve the ``base_url`` for the OpenAI client. + + Selection precedence (first hit wins): + + 1. ``OPENAI_BASE_URL`` env var + 2. ``image_gen.openai.base_url`` in ``config.yaml`` + + Returns ``None`` when neither is set, so ``openai.OpenAI()`` uses its + default (api.openai.com). + """ + env_base_url = _non_empty(os.environ.get("OPENAI_BASE_URL")) + if env_base_url: + return env_base_url + + cfg = _load_openai_config() + openai_cfg = cfg.get("openai") if isinstance(cfg.get("openai"), dict) else {} + if isinstance(openai_cfg, dict): + value = _non_empty(openai_cfg.get("base_url")) + if value: + return value + + return None + + # --------------------------------------------------------------------------- # Source-image loading (for image-to-image / edit) # --------------------------------------------------------------------------- @@ -257,9 +352,28 @@ def generate( aspect_ratio=aspect, ) - tier_id, meta = _resolve_model() + requested_model = _non_empty(kwargs.get("model")) + tier_id, meta = _resolve_model(requested_model) + api_model = _resolve_api_model(requested_model) + base_url = _resolve_base_url() size = _SIZES.get(aspect, _SIZES["square"]) + client_kwargs: Dict[str, Any] = {} + if base_url: + client_kwargs["base_url"] = base_url + try: + client = openai.OpenAI(**client_kwargs) + except Exception as exc: + logger.debug("OpenAI client initialization failed", exc_info=True) + return error_response( + error=f"OpenAI client initialization failed: {exc}", + error_type="api_error", + provider="openai", + model=tier_id, + prompt=prompt, + aspect_ratio=aspect, + ) + # Collect source images (primary + references) for image-to-image. sources: List[str] = [] if isinstance(image_url, str) and image_url.strip(): @@ -270,8 +384,6 @@ def generate( is_edit = bool(sources) modality = "image" if is_edit else "text" - client = openai.OpenAI() - if is_edit: # images.edit() expects file-like objects. Download/read each # source into a named BytesIO so the SDK sends correct multipart. @@ -296,7 +408,7 @@ def generate( try: response = client.images.edit( - model=API_MODEL, + model=api_model, image=files if len(files) > 1 else files[0], prompt=prompt, size=size, # type: ignore[arg-type] # _SIZES values are valid gpt-image sizes @@ -317,7 +429,7 @@ def generate( # gpt-image-2 returns b64_json unconditionally and REJECTS # ``response_format`` as an unknown parameter. Don't send it. payload: Dict[str, Any] = { - "model": API_MODEL, + "model": api_model, "prompt": prompt, "size": size, "n": 1, diff --git a/tests/plugins/image_gen/test_openai_provider.py b/tests/plugins/image_gen/test_openai_provider.py index 2ac61c54ed95c..50bf3ccaf5427 100644 --- a/tests/plugins/image_gen/test_openai_provider.py +++ b/tests/plugins/image_gen/test_openai_provider.py @@ -32,6 +32,12 @@ def _fake_response(*, b64=None, url=None, revised_prompt=None): @pytest.fixture(autouse=True) def _tmp_hermes_home(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + for key in ( + "OPENAI_IMAGE_API_MODEL", + "OPENAI_IMAGE_MODEL", + "OPENAI_BASE_URL", + ): + monkeypatch.delenv(key, raising=False) yield tmp_path @@ -120,6 +126,60 @@ def test_config_top_level_model(self, tmp_path): assert model_id == "gpt-image-2-high" assert meta["quality"] == "high" + def test_requested_tier_from_dispatcher(self): + model_id, meta = openai_plugin._resolve_model("gpt-image-2-low") + assert model_id == "gpt-image-2-low" + assert meta["quality"] == "low" + + def test_requested_non_tier_model_does_not_change_quality_tier(self): + model_id, meta = openai_plugin._resolve_model("third-party-image-model") + assert model_id == openai_plugin.DEFAULT_MODEL + assert meta["quality"] == "medium" + + +class TestApiModelResolution: + def test_default_api_model(self): + assert openai_plugin._resolve_api_model() == "gpt-image-2" + + def test_env_var_api_model_override(self, monkeypatch): + monkeypatch.setenv("OPENAI_IMAGE_API_MODEL", "krill-image-model") + assert openai_plugin._resolve_api_model() == "krill-image-model" + + def test_config_openai_api_model(self, tmp_path): + import yaml + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"image_gen": {"openai": {"api_model": "cfg-image-model"}}}) + ) + assert openai_plugin._resolve_api_model() == "cfg-image-model" + + def test_requested_non_tier_model_becomes_api_model(self): + assert openai_plugin._resolve_api_model("third-party-image-model") == "third-party-image-model" + + def test_requested_tier_does_not_become_api_model(self): + assert openai_plugin._resolve_api_model("gpt-image-2-high") == "gpt-image-2" + + def test_top_level_non_tier_config_becomes_api_model(self, tmp_path): + import yaml + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"image_gen": {"model": "third-party-image-model"}}) + ) + assert openai_plugin._resolve_api_model() == "third-party-image-model" + + def test_config_base_url(self, tmp_path): + import yaml + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"image_gen": {"openai": {"base_url": "https://example.test/v1"}}}) + ) + assert openai_plugin._resolve_base_url() == "https://example.test/v1" + + def test_env_base_url_beats_config(self, tmp_path, monkeypatch): + import yaml + monkeypatch.setenv("OPENAI_BASE_URL", "https://env.example.test/v1") + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"image_gen": {"openai": {"base_url": "https://cfg.example.test/v1"}}}) + ) + assert openai_plugin._resolve_base_url() == "https://env.example.test/v1" + # ── Generate ──────────────────────────────────────────────────────────────── @@ -244,6 +304,98 @@ def test_tier_maps_to_quality(self, provider, monkeypatch, tier, expected_qualit # Always the same underlying API model regardless of tier. assert fake_client.images.generate.call_args.kwargs["model"] == "gpt-image-2" + def test_requested_non_tier_model_sets_api_model(self, provider): + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + + with _patched_openai(fake_client): + result = provider.generate("a cat", model="third-party-image-model") + + assert result["model"] == "gpt-image-2-medium" + assert result["quality"] == "medium" + assert fake_client.images.generate.call_args.kwargs["model"] == "third-party-image-model" + assert fake_client.images.generate.call_args.kwargs["quality"] == "medium" + + def test_requested_tier_model_sets_quality_not_api_model(self, provider): + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + + with _patched_openai(fake_client): + result = provider.generate("a cat", model="gpt-image-2-high") + + assert result["model"] == "gpt-image-2-high" + assert result["quality"] == "high" + assert fake_client.images.generate.call_args.kwargs["model"] == "gpt-image-2" + assert fake_client.images.generate.call_args.kwargs["quality"] == "high" + + def test_configured_base_url_passed_to_openai_client(self, provider, tmp_path): + import yaml + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"image_gen": {"openai": {"base_url": "https://cfg.example.test/v1"}}}) + ) + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + fake_openai = MagicMock() + fake_openai.OpenAI.return_value = fake_client + + with patch.dict("sys.modules", {"openai": fake_openai}): + provider.generate("a cat") + + fake_openai.OpenAI.assert_called_once_with(base_url="https://cfg.example.test/v1") + + def test_client_initialization_error_returns_error_response(self, provider): + fake_openai = MagicMock() + fake_openai.OpenAI.side_effect = ValueError("invalid base URL") + + with patch.dict("sys.modules", {"openai": fake_openai}): + result = provider.generate("a cat") + + assert result["success"] is False + assert result["error_type"] == "api_error" + assert "invalid base URL" in result["error"] + + def test_edit_uses_configured_api_model_and_base_url( + self, provider, tmp_path, monkeypatch + ): + import yaml + + (tmp_path / "config.yaml").write_text( + yaml.safe_dump( + { + "image_gen": { + "openai": { + "api_model": "configured-edit-model", + "base_url": "https://images.example.test/v1", + } + } + } + ) + ) + source = tmp_path / "source.png" + source.write_bytes(bytes.fromhex(_PNG_HEX)) + + fake_client = MagicMock() + fake_client.images.edit.return_value = _fake_response(b64=_b64_png()) + fake_openai = MagicMock() + fake_openai.OpenAI.return_value = fake_client + + with patch.dict("sys.modules", {"openai": fake_openai}): + result = provider.generate( + "edit this cat", + image_url=str(source), + model="dispatcher-image-model", + ) + + assert result["success"] is True + assert result["modality"] == "image" + assert result["model"] == openai_plugin.DEFAULT_MODEL + assert result["quality"] == "medium" + fake_openai.OpenAI.assert_called_once_with( + base_url="https://images.example.test/v1" + ) + assert fake_client.images.edit.call_args.kwargs["model"] == "configured-edit-model" + fake_client.images.generate.assert_not_called() + @pytest.mark.parametrize("aspect,expected_size", [ ("landscape", "1536x1024"), ("square", "1024x1024"), diff --git a/tests/tools/test_image_generation_plugin_dispatch.py b/tests/tools/test_image_generation_plugin_dispatch.py index f96da8d64df4b..6f86008764d19 100644 --- a/tests/tools/test_image_generation_plugin_dispatch.py +++ b/tests/tools/test_image_generation_plugin_dispatch.py @@ -30,6 +30,26 @@ def generate(self, prompt, aspect_ratio="landscape", **kwargs): } +class _CapturingCodexProvider(ImageGenProvider): + def __init__(self): + self.calls = [] + + @property + def name(self) -> str: + return "codex" + + def generate(self, prompt, aspect_ratio="landscape", **kwargs): + self.calls.append({"prompt": prompt, "aspect_ratio": aspect_ratio, **kwargs}) + return { + "success": True, + "image": "/tmp/codex-test.png", + "model": kwargs.get("model", "default-model"), + "prompt": prompt, + "aspect_ratio": aspect_ratio, + "provider": "codex", + } + + class TestPluginDispatch: def test_dispatch_routes_to_codex_provider(self, monkeypatch, tmp_path): from tools import image_generation_tool @@ -52,6 +72,34 @@ def test_dispatch_routes_to_codex_provider(self, monkeypatch, tmp_path): assert payload["image"] == "/tmp/codex-test.png" assert payload["aspect_ratio"] == "square" + def test_dispatch_passes_configured_model_to_plugin(self, monkeypatch, tmp_path): + from tools import image_generation_tool + from agent import image_gen_registry as registry_module + from hermes_cli import plugins as plugins_module + + provider = _CapturingCodexProvider() + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text( + "image_gen:\n provider: codex\n model: third-party-image-model\n" + ) + + monkeypatch.setattr(image_generation_tool, "_read_configured_image_provider", lambda: "codex") + monkeypatch.setattr(plugins_module, "_ensure_plugins_discovered", lambda: None) + monkeypatch.setattr(registry_module, "get_provider", lambda name: provider if name == "codex" else None) + + dispatched = image_generation_tool._dispatch_to_plugin_provider("draw cat", "square") + payload = json.loads(dispatched) + + assert payload["success"] is True + assert payload["model"] == "third-party-image-model" + assert provider.calls == [ + { + "prompt": "draw cat", + "aspect_ratio": "square", + "model": "third-party-image-model", + } + ] + def test_dispatch_reports_missing_registered_provider(self, monkeypatch, tmp_path): from tools import image_generation_tool from hermes_cli import plugins as plugins_module