diff --git a/plugins/example-plugin/tests/integration/conftest.py b/plugins/example-plugin/tests/integration/conftest.py index 70aa809933..9c26bea06a 100644 --- a/plugins/example-plugin/tests/integration/conftest.py +++ b/plugins/example-plugin/tests/integration/conftest.py @@ -1,15 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Pytest fixture re-export for the example plugin integration tests. +"""Pytest fixture re-exports for the example plugin integration tests. -Re-exporting :func:`igw_plugin_harness` from a project-level ``conftest.py`` -is the standard pytest pattern for sharing a fixture across a test package -without importing it at the top of every test module. Listing it in -``__all__`` makes the re-export explicit so it isn't flagged as an unused -import. +The module-scope helpers (``_igw_app_context``, ``_igw_extra_services``) +are re-imported so pytest can resolve :func:`igw_plugin_harness`'s +dependency chain. The default empty ``_igw_extra_services`` tuple +applies — no services beyond IGW + Models are mounted. """ -from nmp.core.inference_gateway.testing.fixtures import igw_plugin_harness +from nmp.core.inference_gateway.testing.fixtures import ( + _igw_app_context, + _igw_extra_services, + igw_plugin_harness, +) -__all__ = ["igw_plugin_harness"] +__all__ = [ + "_igw_app_context", + "_igw_extra_services", + "igw_plugin_harness", +] diff --git a/plugins/example-plugin/tests/integration/test_inference_middleware.py b/plugins/example-plugin/tests/integration/test_inference_middleware.py index 5f48bc03de..b6b4da34c4 100644 --- a/plugins/example-plugin/tests/integration/test_inference_middleware.py +++ b/plugins/example-plugin/tests/integration/test_inference_middleware.py @@ -34,7 +34,6 @@ pytestmark = [pytest.mark.integration] -DEFAULT_WORKSPACE = "default" EXAMPLE_PLUGIN_NAME = "nemo-example-middleware" EXAMPLE_PLUGIN_CONFIG_TYPE = ExampleMiddlewareConfig.__entity_type__ @@ -77,21 +76,21 @@ def test_safe_input_is_proxied_to_backend(self, igw_plugin_harness: IGWPluginHar ], ) h.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=f"example-provider-{test_id}", served_models={model_name: model_name}, ) with h.use_plugin(EXAMPLE_PLUGIN_NAME, ExampleInferenceMiddleware()): h.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=virtual_model_name, - default_model_entity=f"{DEFAULT_WORKSPACE}/{model_name}", + default_model_entity=f"{h.workspace}/{model_name}", request_middleware=[_build_middleware_call(blocked_keywords=["violence"])], ) response = h.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, body={ "model": virtual_model_name, "messages": [{"role": "user", "content": "Tell me about flowers."}], @@ -109,16 +108,16 @@ def test_blocked_input_short_circuits_proxy(self, igw_plugin_harness: IGWPluginH block_message = "That topic is off-limits." h.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=f"example-provider-{test_id}", served_models={model_name: model_name}, ) with h.use_plugin(EXAMPLE_PLUGIN_NAME, ExampleInferenceMiddleware()): h.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=virtual_model_name, - default_model_entity=f"{DEFAULT_WORKSPACE}/{model_name}", + default_model_entity=f"{h.workspace}/{model_name}", request_middleware=[ _build_middleware_call( blocked_keywords=["violence"], @@ -128,7 +127,7 @@ def test_blocked_input_short_circuits_proxy(self, igw_plugin_harness: IGWPluginH ) response = h.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, body={ "model": virtual_model_name, "messages": [{"role": "user", "content": "Tell me about violence."}], @@ -171,21 +170,21 @@ def test_backend_response_is_redacted(self, igw_plugin_harness: IGWPluginHarness ], ) h.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=f"example-provider-{test_id}", served_models={model_name: model_name}, ) with h.use_plugin(EXAMPLE_PLUGIN_NAME, ExampleInferenceMiddleware()): h.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=virtual_model_name, - default_model_entity=f"{DEFAULT_WORKSPACE}/{model_name}", + default_model_entity=f"{h.workspace}/{model_name}", response_middleware=[_build_middleware_call(blocked_keywords=["secret"])], ) response = h.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, body={ "model": virtual_model_name, "messages": [{"role": "user", "content": "Share the answer."}], @@ -233,21 +232,21 @@ def test_streaming_backend_response_is_redacted(self, igw_plugin_harness: IGWPlu ], ) h.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=f"example-stream-provider-{test_id}", served_models={model_name: model_name}, ) with h.use_plugin(EXAMPLE_PLUGIN_NAME, ExampleInferenceMiddleware()): h.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, name=virtual_model_name, - default_model_entity=f"{DEFAULT_WORKSPACE}/{model_name}", + default_model_entity=f"{h.workspace}/{model_name}", response_middleware=[_build_middleware_call(blocked_keywords=["secret"])], ) chunks = h.stream_chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=h.workspace, body={ "model": virtual_model_name, "messages": [{"role": "user", "content": "Share the answer."}], diff --git a/plugins/nemo-guardrails/tests/integration/conftest.py b/plugins/nemo-guardrails/tests/integration/conftest.py index 6043078be0..315321bb07 100644 --- a/plugins/nemo-guardrails/tests/integration/conftest.py +++ b/plugins/nemo-guardrails/tests/integration/conftest.py @@ -3,23 +3,51 @@ """Pytest fixtures for the guardrails plugin integration tests. -Re-exports the IGW harness fixtures so test modules don't have to import -them at the top of every file, and provides an autouse fixture that -keeps ``nemoguardrails`` from reaching out to HuggingFace at startup. - -- :func:`igw_plugin_harness` — default; no real port for IGW. -- :func:`igw_loopback_harness` — opt-in; IGW additionally bound on a real - ``127.0.0.1:`` for tests that need IGW's loopback URL. Call it with - extra services to mount additional routes. +Re-exports the IGW harness fixtures so test modules don't have to +import them. ``_igw_extra_services`` is overridden below to mount +:class:`GuardrailsService` on the module-scoped app — entity-backed +guardrail-config tests need its CRUD routes. The module-scope helpers +(``_igw_app_context``, ``_igw_loopback_context``) are re-imported so +pytest can resolve the dependency chain from this conftest's scope. + +``HF_HUB_OFFLINE`` is set at conftest import time, **before** the +``GuardrailsService`` import below — importing ``GuardrailsService`` +transitively imports ``nemoguardrails``, which reaches HuggingFace at +import time if not told to stay offline. A function-scoped autouse +``monkeypatch`` fixture would be too late: it doesn't run until after +the module-scoped fixture setup that triggers these imports. """ +import os + import pytest -from nmp.core.inference_gateway.testing.fixtures import igw_loopback_harness, igw_plugin_harness +from nmp.core.inference_gateway.testing.fixtures import ( + _igw_app_context, + _igw_loopback_context, + igw_loopback_harness, + igw_plugin_harness, +) +from nmp.testing.client import ServiceFactory + +# Must precede the ``nemoguardrails``-pulling import below. +os.environ.setdefault("HF_HUB_OFFLINE", "1") + +from nmp.guardrails.service import GuardrailsService # noqa: E402 + +__all__ = [ + "_igw_app_context", + "_igw_loopback_context", + "igw_loopback_harness", + "igw_plugin_harness", +] -__all__ = ["igw_loopback_harness", "igw_plugin_harness"] +@pytest.fixture(scope="module") +def _igw_extra_services() -> tuple[ServiceFactory, ...]: + """Mount :class:`GuardrailsService` on the module-scoped IGW + Models app. -@pytest.fixture(autouse=True) -def offline_huggingface(monkeypatch: pytest.MonkeyPatch) -> None: - """Skip ``nemoguardrails`` HuggingFace tokenizer downloads — they time out offline.""" - monkeypatch.setenv("HF_HUB_OFFLINE", "1") + Every integration test here gets Guardrails CRUD routes whether it + uses them or not — the startup cost amortises across the module, + so files that only touch inline configs pay almost nothing extra. + """ + return (GuardrailsService,) diff --git a/plugins/nemo-guardrails/tests/integration/test_content_safety_rails.py b/plugins/nemo-guardrails/tests/integration/test_content_safety_rails.py index 1a3dfd7ba6..751f044e0c 100644 --- a/plugins/nemo-guardrails/tests/integration/test_content_safety_rails.py +++ b/plugins/nemo-guardrails/tests/integration/test_content_safety_rails.py @@ -16,7 +16,6 @@ from nmp.testing.mock_chat_completions import ChatCompletion, chat_completion from .utils import ( - DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, RailType, make_guardrail_config, @@ -89,9 +88,20 @@ class ContentSafetyTestDataNames: content_safety_entity_ref: str -def _make_test_data_names(*, main_model_prefix: str = "main-model") -> ContentSafetyTestDataNames: - base_test_data_names = make_guardrails_test_data_names(main_model_prefix=main_model_prefix) - content_safety_model = make_served_model(test_id=base_test_data_names.test_id, prefix="cs-model") +def _make_test_data_names( + *, + main_model_prefix: str = "main-model", + workspace: str, +) -> ContentSafetyTestDataNames: + base_test_data_names = make_guardrails_test_data_names( + main_model_prefix=main_model_prefix, + workspace=workspace, + ) + content_safety_model = make_served_model( + test_id=base_test_data_names.test_id, + prefix="cs-model", + workspace=workspace, + ) return ContentSafetyTestDataNames( main_model_served_name=base_test_data_names.main_model_served_name, @@ -237,7 +247,7 @@ def test_input_rail( :meth:`test_resolver_fills_content_safety_base_url`. """ harness = igw_plugin_harness - test_data_names = _make_test_data_names() + test_data_names = _make_test_data_names(workspace=harness.workspace) user_input = self.UNSAFE_USER_INPUT if expect_blocked else self.USER_INPUT content_safety_response = ( @@ -256,7 +266,7 @@ def test_input_rail( responses=[ChatCompletion(body=chat_completion(content=self.BACKEND_RESPONSE))], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={ test_data_names.main_model_served_name: test_data_names.main_model_served_name, @@ -265,13 +275,13 @@ def test_input_rail( ) # Passthrough VM so body["model"] (main entity ref) resolves to the mock NIM URL. harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data( rail_types=[RailType.INPUT], @@ -281,13 +291,13 @@ def test_input_rail( ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, request_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": user_input}], @@ -342,7 +352,7 @@ def test_output_rail( :meth:`test_resolver_fills_content_safety_base_url`. """ harness = igw_plugin_harness - test_data_names = _make_test_data_names() + test_data_names = _make_test_data_names(workspace=harness.workspace) content_safety_response = ( self._unsafe_output_content_safety_response() if expect_blocked @@ -358,7 +368,7 @@ def test_output_rail( responses=[ChatCompletion(body=chat_completion(content=content_safety_response))], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={ test_data_names.main_model_served_name: test_data_names.main_model_served_name, @@ -366,13 +376,13 @@ def test_output_rail( }, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data( rail_types=[RailType.OUTPUT], @@ -382,13 +392,13 @@ def test_output_rail( ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, response_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], @@ -438,7 +448,7 @@ def test_input_and_output_rails( reaches the caller intact. """ harness = igw_plugin_harness - test_data_names = _make_test_data_names() + test_data_names = _make_test_data_names(workspace=harness.workspace) input_verdict = ( self._unsafe_input_content_safety_response() @@ -466,7 +476,7 @@ def test_input_and_output_rails( responses=[ChatCompletion(body=chat_completion(content=self.BACKEND_RESPONSE))], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={ test_data_names.main_model_served_name: test_data_names.main_model_served_name, @@ -474,13 +484,13 @@ def test_input_and_output_rails( }, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data( rail_types=[RailType.INPUT, RailType.OUTPUT], @@ -490,14 +500,14 @@ def test_input_and_output_rails( ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, request_middleware=[make_middleware_call(guardrail_config)], response_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], @@ -559,7 +569,7 @@ def test_resolver_fills_content_safety_base_url( covered by :meth:`test_input_rail` / :meth:`test_output_rail`. """ harness = igw_loopback_harness() - test_data_names = _make_test_data_names(main_model_prefix="gr-main") + test_data_names = _make_test_data_names(main_model_prefix="gr-main", workspace=harness.workspace) harness.mock_chat_completions( test_data_names.content_safety_model_served_name, @@ -570,7 +580,7 @@ def test_resolver_fills_content_safety_base_url( responses=[ChatCompletion(body=chat_completion(content=self.BACKEND_RESPONSE))], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={ test_data_names.main_model_served_name: test_data_names.main_model_served_name, @@ -578,18 +588,18 @@ def test_resolver_fills_content_safety_base_url( }, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.content_safety_model_served_name, default_model_entity=test_data_names.content_safety_entity_ref, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data( rail_types=[RailType.INPUT], @@ -599,13 +609,13 @@ def test_resolver_fills_content_safety_base_url( ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, request_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], diff --git a/plugins/nemo-guardrails/tests/integration/test_injection_detection_rails.py b/plugins/nemo-guardrails/tests/integration/test_injection_detection_rails.py index eee5bd8c9a..1b26a7543d 100644 --- a/plugins/nemo-guardrails/tests/integration/test_injection_detection_rails.py +++ b/plugins/nemo-guardrails/tests/integration/test_injection_detection_rails.py @@ -15,10 +15,9 @@ from nemo_guardrails_plugin.constants import GUARDRAILS_PLUGIN_CONFIG_TYPE from nemo_platform.types.inference.middleware_call_param import MiddlewareCallParam from nmp.core.inference_gateway.testing.harness import IGWLoopbackHarness -from nmp.guardrails.service import GuardrailsService from nmp.testing.mock_chat_completions import ChatCompletion, chat_completion -from .utils import DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, GuardrailsTestDataNames, make_guardrails_test_data_names +from .utils import GUARDRAILS_PLUGIN_NAME, GuardrailsTestDataNames, make_guardrails_test_data_names pytestmark = [pytest.mark.integration] @@ -41,17 +40,17 @@ class TestInjectionDetection: REFUSAL_PREFIX = "I'm sorry, the desired output triggered rule(s) designed to mitigate exploitation of" @staticmethod - def _middleware_call(config_name: str) -> MiddlewareCallParam: + def _middleware_call(workspace: str, config_name: str) -> MiddlewareCallParam: return { "name": GUARDRAILS_PLUGIN_NAME, "config_type": GUARDRAILS_PLUGIN_CONFIG_TYPE, - "config_id": f"{DEFAULT_WORKSPACE}/{config_name}", + "config_id": f"{workspace}/{config_name}", } @staticmethod def _delete_config_if_present(harness: IGWLoopbackHarness, config_name: str) -> None: try: - harness.sdk.guardrail.configs.delete(name=config_name, workspace=DEFAULT_WORKSPACE) + harness.sdk.guardrail.configs.delete(name=config_name, workspace=harness.workspace) except nemo_platform.NotFoundError: pass @@ -99,37 +98,40 @@ def _setup_entity_backed_vm( *, config_data: dict[str, Any], ) -> GuardrailsTestDataNames: - test_data_names = make_guardrails_test_data_names(main_model_prefix="main-model") + test_data_names = make_guardrails_test_data_names( + main_model_prefix="main-model", + workspace=harness.workspace, + ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) harness.sdk.guardrail.configs.create( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.guardrail_config_name, description="Entity-backed injection detection config for integration tests", data=config_data, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, - response_middleware=[self._middleware_call(test_data_names.guardrail_config_name)], + response_middleware=[self._middleware_call(harness.workspace, test_data_names.guardrail_config_name)], ) return test_data_names def _chat_completions(self, harness: IGWLoopbackHarness, *, model: str) -> dict[str, Any]: return harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ - "model": f"{DEFAULT_WORKSPACE}/{model}", + "model": f"{harness.workspace}/{model}", "messages": [{"role": "user", "content": self.USER_INPUT}], }, ) @@ -153,7 +155,7 @@ def _assert_inference_uses_config( harness.assert_called_once(test_data_names.main_model_served_name) guardrails_data: dict[str, Any] = response.get("guardrails_data") or {} - assert guardrails_data.get("config_ids") == [f"{DEFAULT_WORKSPACE}/{test_data_names.guardrail_config_name}"] + assert guardrails_data.get("config_ids") == [f"{harness.workspace}/{test_data_names.guardrail_config_name}"] if expected_blocked_content is not None: assert response["choices"][0]["finish_reason"] == "content_filter" @@ -175,7 +177,7 @@ def test_builtin_injection_detection( expect_blocked: bool, ) -> None: """Built-in injection detection should reject unsafe model output and allow safe output.""" - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): test_data_names = self._setup_entity_backed_vm( @@ -206,7 +208,7 @@ def test_custom_yara_rule( expect_blocked: bool, ) -> None: """Custom inline YARA rules should reject matching model output and allow non-matches.""" - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): test_data_names = self._setup_entity_backed_vm( diff --git a/plugins/nemo-guardrails/tests/integration/test_middleware_config_caching.py b/plugins/nemo-guardrails/tests/integration/test_middleware_config_caching.py index 1b5da3a3fe..ab369cc23e 100644 --- a/plugins/nemo-guardrails/tests/integration/test_middleware_config_caching.py +++ b/plugins/nemo-guardrails/tests/integration/test_middleware_config_caching.py @@ -16,11 +16,9 @@ from nemo_guardrails_plugin.constants import GUARDRAILS_PLUGIN_CONFIG_TYPE from nemo_platform.types.inference.middleware_call_param import MiddlewareCallParam from nmp.core.inference_gateway.testing.harness import IGWLoopbackHarness, IGWPluginHarness -from nmp.guardrails.service import GuardrailsService from nmp.testing.mock_chat_completions import ChatCompletion, chat_completion from .utils import ( - DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, GuardrailsTestDataNames, make_guardrails_test_data_names, @@ -98,17 +96,17 @@ def _backend_response() -> str: return "Paris is the capital of France." @staticmethod - def _middleware_call(config_name: str) -> MiddlewareCallParam: + def _middleware_call(workspace: str, config_name: str) -> MiddlewareCallParam: return { "name": GUARDRAILS_PLUGIN_NAME, "config_type": GUARDRAILS_PLUGIN_CONFIG_TYPE, - "config_id": f"{DEFAULT_WORKSPACE}/{config_name}", + "config_id": f"{workspace}/{config_name}", } @staticmethod def _delete_config_if_present(harness: IGWPluginHarness, config_name: str) -> None: try: - harness.sdk.guardrail.configs.delete(name=config_name, workspace=DEFAULT_WORKSPACE) + harness.sdk.guardrail.configs.delete(name=config_name, workspace=harness.workspace) except nemo_platform.NotFoundError: pass @@ -127,7 +125,7 @@ def _assert_chat_completions_unavailable( ) -> None: with pytest.raises(nemo_platform.APIStatusError) as exc_info: harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": model, "messages": [{"role": "user", "content": cls.USER_INPUT}], @@ -154,27 +152,30 @@ def _setup_entity_backed_vm( *, config_version: str, ) -> GuardrailsTestDataNames: - test_data_names = make_guardrails_test_data_names(main_model_prefix="main-model") + test_data_names = make_guardrails_test_data_names( + main_model_prefix="main-model", + workspace=harness.workspace, + ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) harness.sdk.guardrail.configs.create( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.guardrail_config_name, description="Entity-backed self-check config for middleware cache tests", data=self._config_data(version=config_version, main_base_url=harness.nim_base_url), ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, - request_middleware=[self._middleware_call(test_data_names.guardrail_config_name)], - response_middleware=[self._middleware_call(test_data_names.guardrail_config_name)], + request_middleware=[self._middleware_call(harness.workspace, test_data_names.guardrail_config_name)], + response_middleware=[self._middleware_call(harness.workspace, test_data_names.guardrail_config_name)], ) return test_data_names @@ -207,7 +208,7 @@ def _assert_inference_uses_config( ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body=self._chat_completions_body(model=test_data_names.request_virtual_model_name), ) @@ -234,7 +235,7 @@ def test_entity_config_update_reflected_in_next_request( igw_loopback_harness: Callable[..., IGWLoopbackHarness], ) -> None: """A refreshed entity-backed config should affect the next real Guardrails execution.""" - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): test_data_names = self._setup_entity_backed_vm(harness, config_version="v1") @@ -249,7 +250,7 @@ def test_entity_config_update_reflected_in_next_request( # Update the config referenced by the VirtualModel harness.sdk.guardrail.configs.update( name=test_data_names.guardrail_config_name, - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, data=self._config_data(version="v2", main_base_url=harness.nim_base_url), ) @@ -279,7 +280,7 @@ def test_entity_config_delete_blocks_next_request( igw_loopback_harness: Callable[..., IGWLoopbackHarness], ) -> None: """Deleting a referenced config should fail closed after IGW refreshes middleware config refs.""" - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): test_data_names = self._setup_entity_backed_vm(harness, config_version="v1") @@ -294,7 +295,7 @@ def test_entity_config_delete_blocks_next_request( # Delete the config referenced by the VirtualModel. harness.sdk.guardrail.configs.delete( name=test_data_names.guardrail_config_name, - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, ) self._refresh_caches(harness) @@ -312,7 +313,7 @@ def test_entity_config_recreate_reflected_in_next_request( igw_loopback_harness: Callable[..., IGWLoopbackHarness], ) -> None: """Recreating the same config_id should let the next refresh recover the failing VM.""" - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): test_data_names = self._setup_entity_backed_vm(harness, config_version="v1") @@ -320,7 +321,7 @@ def test_entity_config_recreate_reflected_in_next_request( # Delete the config referenced by the VirtualModel. harness.sdk.guardrail.configs.delete( name=test_data_names.guardrail_config_name, - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, ) self._refresh_caches(harness) @@ -333,7 +334,7 @@ def test_entity_config_recreate_reflected_in_next_request( # Recreate the config referenced by the VirtualModel. harness.sdk.guardrail.configs.create( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.guardrail_config_name, description="Recreated self-check config for middleware cache tests", data=self._config_data(version="v2", main_base_url=harness.nim_base_url), diff --git a/plugins/nemo-guardrails/tests/integration/test_multimodal_rails.py b/plugins/nemo-guardrails/tests/integration/test_multimodal_rails.py index 6685046787..9c342fd9aa 100644 --- a/plugins/nemo-guardrails/tests/integration/test_multimodal_rails.py +++ b/plugins/nemo-guardrails/tests/integration/test_multimodal_rails.py @@ -16,11 +16,9 @@ from nemo_guardrails_plugin.constants import GUARDRAILS_PLUGIN_CONFIG_TYPE from nemo_platform.types.inference.middleware_call_param import MiddlewareCallParam from nmp.core.inference_gateway.testing.harness import IGWLoopbackHarness -from nmp.guardrails.service import GuardrailsService from nmp.testing.mock_chat_completions import ChatCompletion, chat_completion from .utils import ( - DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, make_guardrails_test_data_names, make_served_model, @@ -40,9 +38,20 @@ class MultimodalTestDataNames: vision_model_entity_ref: str -def _make_test_data_names(*, main_model_prefix: str = "main-model") -> MultimodalTestDataNames: - base_test_data_names = make_guardrails_test_data_names(main_model_prefix=main_model_prefix) - vision_model = make_served_model(test_id=base_test_data_names.test_id, prefix="vision-model") +def _make_test_data_names( + *, + main_model_prefix: str = "main-model", + workspace: str, +) -> MultimodalTestDataNames: + base_test_data_names = make_guardrails_test_data_names( + main_model_prefix=main_model_prefix, + workspace=workspace, + ) + vision_model = make_served_model( + test_id=base_test_data_names.test_id, + prefix="vision-model", + workspace=workspace, + ) return MultimodalTestDataNames( main_model_served_name=base_test_data_names.main_model_served_name, @@ -99,17 +108,17 @@ class TestMultimodalContentSafety: } @staticmethod - def _middleware_call(config_name: str) -> MiddlewareCallParam: + def _middleware_call(workspace: str, config_name: str) -> MiddlewareCallParam: return { "name": GUARDRAILS_PLUGIN_NAME, "config_type": GUARDRAILS_PLUGIN_CONFIG_TYPE, - "config_id": f"{DEFAULT_WORKSPACE}/{config_name}", + "config_id": f"{workspace}/{config_name}", } @staticmethod def _delete_config_if_present(harness: IGWLoopbackHarness, config_name: str) -> None: try: - harness.sdk.guardrail.configs.delete(name=config_name, workspace=DEFAULT_WORKSPACE) + harness.sdk.guardrail.configs.delete(name=config_name, workspace=harness.workspace) except nemo_platform.NotFoundError: pass @@ -143,10 +152,10 @@ def _user_message(cls, text: str) -> dict[str, Any]: } def _setup_entity_backed_vm(self, harness: IGWLoopbackHarness) -> MultimodalTestDataNames: - test_data_names = _make_test_data_names() + test_data_names = _make_test_data_names(workspace=harness.workspace) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={ test_data_names.main_model_served_name: test_data_names.main_model_served_name, @@ -154,7 +163,7 @@ def _setup_entity_backed_vm(self, harness: IGWLoopbackHarness) -> MultimodalTest }, ) harness.sdk.guardrail.configs.create( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.guardrail_config_name, description="Entity-backed multimodal input rail config for integration tests", data=self._config_data( @@ -163,15 +172,15 @@ def _setup_entity_backed_vm(self, harness: IGWLoopbackHarness) -> MultimodalTest ), ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, - request_middleware=[self._middleware_call(test_data_names.guardrail_config_name)], + request_middleware=[self._middleware_call(harness.workspace, test_data_names.guardrail_config_name)], ) return test_data_names @@ -197,9 +206,9 @@ def _assert_inference_uses_config( ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ - "model": f"{DEFAULT_WORKSPACE}/{test_data_names.request_virtual_model_name}", + "model": f"{harness.workspace}/{test_data_names.request_virtual_model_name}", "messages": [self._user_message(user_text)], }, ) @@ -209,7 +218,7 @@ def _assert_inference_uses_config( harness.assert_request_messages_contain(test_data_names.vision_model_entity_ref, user_text) harness.assert_request_messages_contain(test_data_names.vision_model_entity_ref, self.IMAGE_DATA_URL) guardrails_data: dict[str, Any] = response.get("guardrails_data") or {} - assert guardrails_data.get("config_ids") == [f"{DEFAULT_WORKSPACE}/{test_data_names.guardrail_config_name}"] + assert guardrails_data.get("config_ids") == [f"{harness.workspace}/{test_data_names.guardrail_config_name}"] if expect_blocked: harness.assert_no_calls_to(test_data_names.main_model_served_name) @@ -237,7 +246,7 @@ def test_multimodal_input_rail( expect_blocked: bool, ) -> None: """Vision input rails should receive the user's message and image data before the backend runs.""" - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): test_data_names = self._setup_entity_backed_vm(harness) diff --git a/plugins/nemo-guardrails/tests/integration/test_parallel_rails.py b/plugins/nemo-guardrails/tests/integration/test_parallel_rails.py index 37d04cc4e2..4411cdde6a 100644 --- a/plugins/nemo-guardrails/tests/integration/test_parallel_rails.py +++ b/plugins/nemo-guardrails/tests/integration/test_parallel_rails.py @@ -17,11 +17,9 @@ from nemo_guardrails_plugin.constants import GUARDRAILS_PLUGIN_CONFIG_TYPE from nemo_platform.types.inference.middleware_call_param import MiddlewareCallParam from nmp.core.inference_gateway.testing.harness import IGWLoopbackHarness -from nmp.guardrails.service import GuardrailsService from nmp.testing.mock_chat_completions import ChatCompletion, chat_completion from .utils import ( - DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, make_guardrails_test_data_names, make_served_model, @@ -43,10 +41,25 @@ class ParallelRailsTestDataNames: topic_control_model_entity_ref: str -def _make_test_data_names(*, main_model_prefix: str = "main-model") -> ParallelRailsTestDataNames: - base_test_data_names = make_guardrails_test_data_names(main_model_prefix=main_model_prefix) - content_safety_model = make_served_model(test_id=base_test_data_names.test_id, prefix="cs-model") - topic_control_model = make_served_model(test_id=base_test_data_names.test_id, prefix="tc-model") +def _make_test_data_names( + *, + main_model_prefix: str = "main-model", + workspace: str, +) -> ParallelRailsTestDataNames: + base_test_data_names = make_guardrails_test_data_names( + main_model_prefix=main_model_prefix, + workspace=workspace, + ) + content_safety_model = make_served_model( + test_id=base_test_data_names.test_id, + prefix="cs-model", + workspace=workspace, + ) + topic_control_model = make_served_model( + test_id=base_test_data_names.test_id, + prefix="tc-model", + workspace=workspace, + ) return ParallelRailsTestDataNames( main_model_served_name=base_test_data_names.main_model_served_name, @@ -116,17 +129,17 @@ def _safe_input_topic_control_response() -> str: return "on-topic" @staticmethod - def _middleware_call(config_name: str) -> MiddlewareCallParam: + def _middleware_call(workspace: str, config_name: str) -> MiddlewareCallParam: return { "name": GUARDRAILS_PLUGIN_NAME, "config_type": GUARDRAILS_PLUGIN_CONFIG_TYPE, - "config_id": f"{DEFAULT_WORKSPACE}/{config_name}", + "config_id": f"{workspace}/{config_name}", } @staticmethod def _delete_config_if_present(harness: IGWLoopbackHarness, config_name: str) -> None: try: - harness.sdk.guardrail.configs.delete(name=config_name, workspace=DEFAULT_WORKSPACE) + harness.sdk.guardrail.configs.delete(name=config_name, workspace=harness.workspace) except nemo_platform.NotFoundError: pass @@ -185,10 +198,10 @@ def _setup_entity_backed_vm( *, parallel: bool, ) -> ParallelRailsTestDataNames: - test_data_names = _make_test_data_names() + test_data_names = _make_test_data_names(workspace=harness.workspace) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={ test_data_names.main_model_served_name: test_data_names.main_model_served_name, @@ -197,7 +210,7 @@ def _setup_entity_backed_vm( }, ) harness.sdk.guardrail.configs.create( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.guardrail_config_name, description="Entity-backed parallel rails config for integration tests", data=self._config_data( @@ -208,15 +221,15 @@ def _setup_entity_backed_vm( ), ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, - request_middleware=[self._middleware_call(test_data_names.guardrail_config_name)], + request_middleware=[self._middleware_call(harness.workspace, test_data_names.guardrail_config_name)], ) return test_data_names @@ -239,7 +252,7 @@ def test_input_rails_call_order_when_blocking( as unsafe. In parallel mode, the topic control rail should also still run. In sequential mode, the topic control rail should never run. """ - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): test_data_names = self._setup_entity_backed_vm(harness, parallel=parallel) @@ -256,9 +269,9 @@ def test_input_rails_call_order_when_blocking( ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ - "model": f"{DEFAULT_WORKSPACE}/{test_data_names.request_virtual_model_name}", + "model": f"{harness.workspace}/{test_data_names.request_virtual_model_name}", "messages": [{"role": "user", "content": self.USER_INPUT}], "guardrails": {"options": {"log": {"activated_rails": True}}}, }, diff --git a/plugins/nemo-guardrails/tests/integration/test_self_check_rails.py b/plugins/nemo-guardrails/tests/integration/test_self_check_rails.py index 59d8d13643..c202c39d57 100644 --- a/plugins/nemo-guardrails/tests/integration/test_self_check_rails.py +++ b/plugins/nemo-guardrails/tests/integration/test_self_check_rails.py @@ -15,7 +15,6 @@ from nmp.testing.mock_chat_completions import ChatCompletion, chat_completion from .utils import ( - DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, RailType, make_guardrail_config, @@ -141,7 +140,7 @@ def test_input_rail( :meth:`test_resolver_fills_main_base_url`. """ harness = igw_plugin_harness - test_data_names = make_guardrails_test_data_names() + test_data_names = make_guardrails_test_data_names(workspace=harness.workspace) self_check_response = ( self._unsafe_input_self_check_response() @@ -159,25 +158,25 @@ def test_input_rail( responses=[ChatCompletion(body=chat_completion(content=self.BACKEND_RESPONSE))], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data(rail_types=[RailType.INPUT], main_base_url=harness.nim_base_url), ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, request_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], @@ -222,7 +221,7 @@ def test_output_rail( lives in :meth:`test_resolver_fills_main_base_url`. """ harness = igw_plugin_harness - test_data_names = make_guardrails_test_data_names() + test_data_names = make_guardrails_test_data_names(workspace=harness.workspace) self_check_response = ( self._unsafe_output_self_check_response() if expected_blocked_response @@ -238,25 +237,25 @@ def test_output_rail( responses=[ChatCompletion(body=chat_completion(content=self_check_response))], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data(rail_types=[RailType.OUTPUT], main_base_url=harness.nim_base_url), ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, response_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], @@ -306,7 +305,7 @@ def test_input_and_output_rails( reaches the caller intact. """ harness = igw_plugin_harness - test_data_names = make_guardrails_test_data_names() + test_data_names = make_guardrails_test_data_names(workspace=harness.workspace) input_verdict = ( self._unsafe_input_self_check_response() if input_blocked else self._safe_input_self_check_response() @@ -328,13 +327,13 @@ def test_input_and_output_rails( responses=[ChatCompletion(body=chat_completion(content=self.BACKEND_RESPONSE))], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data( rail_types=[RailType.INPUT, RailType.OUTPUT], @@ -343,14 +342,14 @@ def test_input_and_output_rails( ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, request_middleware=[make_middleware_call(guardrail_config)], response_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], @@ -415,7 +414,10 @@ def test_resolver_fills_main_base_url( covered by :meth:`test_input_rail` / :meth:`test_output_rail`. """ harness = igw_loopback_harness() - test_data_names = make_guardrails_test_data_names(main_model_prefix="gr-main") + test_data_names = make_guardrails_test_data_names( + main_model_prefix="gr-main", + workspace=harness.workspace, + ) # The rail call (resolver-filled URL → IGW loopback → passthrough VM # proxy) and the backend completion both hit the same socket since @@ -428,30 +430,30 @@ def test_resolver_fills_main_base_url( ], ) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data(rail_types=[RailType.INPUT], main_base_url=None), ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, request_middleware=[make_middleware_call(guardrail_config)], ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], diff --git a/plugins/nemo-guardrails/tests/integration/test_streaming_rails.py b/plugins/nemo-guardrails/tests/integration/test_streaming_rails.py index c63dc74936..aab678295f 100644 --- a/plugins/nemo-guardrails/tests/integration/test_streaming_rails.py +++ b/plugins/nemo-guardrails/tests/integration/test_streaming_rails.py @@ -22,7 +22,6 @@ ) from .utils import ( - DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, RailType, make_guardrail_config, @@ -125,7 +124,10 @@ def test_input_rail_streaming( an SSE stream. """ harness = igw_loopback_harness() - test_data_names = make_guardrails_test_data_names(main_model_prefix="gr-main") + test_data_names = make_guardrails_test_data_names( + main_model_prefix="gr-main", + workspace=harness.workspace, + ) self_check_response = ( self._unsafe_input_self_check_response() @@ -141,30 +143,30 @@ def test_input_rail_streaming( harness.mock_chat_completions(test_data_names.main_model_served_name, responses=model_responses) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data(rail_types=[RailType.INPUT]), ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, request_middleware=[make_middleware_call(guardrail_config)], ) response_payload = harness.stream_chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], @@ -218,7 +220,10 @@ def test_output_rail_streaming( token is forwarded, so the caller sees only the error token. """ harness = igw_loopback_harness() - test_data_names = make_guardrails_test_data_names(main_model_prefix="gr-main") + test_data_names = make_guardrails_test_data_names( + main_model_prefix="gr-main", + workspace=harness.workspace, + ) self_check_response = ( self._unsafe_output_self_check_response() @@ -232,18 +237,18 @@ def test_output_rail_streaming( harness.mock_chat_completions(test_data_names.main_model_served_name, responses=model_responses) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={test_data_names.main_model_served_name: test_data_names.main_model_served_name}, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) guardrail_config = make_guardrail_config( - DEFAULT_WORKSPACE, + harness.workspace, test_data_names.guardrail_config_name, data=self._config_data( rail_types=[RailType.OUTPUT], @@ -252,13 +257,13 @@ def test_output_rail_streaming( ) with harness.load_plugin(GUARDRAILS_PLUGIN_NAME): harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, response_middleware=[make_middleware_call(guardrail_config)], ) response_payload = harness.stream_chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ "model": test_data_names.request_virtual_model_name, "messages": [{"role": "user", "content": self.USER_INPUT}], diff --git a/plugins/nemo-guardrails/tests/integration/test_topic_control_rails.py b/plugins/nemo-guardrails/tests/integration/test_topic_control_rails.py index 9f0036dac5..86202377ef 100644 --- a/plugins/nemo-guardrails/tests/integration/test_topic_control_rails.py +++ b/plugins/nemo-guardrails/tests/integration/test_topic_control_rails.py @@ -16,11 +16,9 @@ from nemo_guardrails_plugin.constants import GUARDRAILS_PLUGIN_CONFIG_TYPE from nemo_platform.types.inference.middleware_call_param import MiddlewareCallParam from nmp.core.inference_gateway.testing.harness import IGWLoopbackHarness -from nmp.guardrails.service import GuardrailsService from nmp.testing.mock_chat_completions import ChatCompletion, chat_completion from .utils import ( - DEFAULT_WORKSPACE, GUARDRAILS_PLUGIN_NAME, RailType, make_guardrails_test_data_names, @@ -41,9 +39,20 @@ class TopicControlTestDataNames: topic_control_entity_ref: str -def _make_test_data_names(*, main_model_prefix: str = "main-model") -> TopicControlTestDataNames: - base_test_data_names = make_guardrails_test_data_names(main_model_prefix=main_model_prefix) - topic_control_model = make_served_model(test_id=base_test_data_names.test_id, prefix="tc-model") +def _make_test_data_names( + *, + main_model_prefix: str = "main-model", + workspace: str, +) -> TopicControlTestDataNames: + base_test_data_names = make_guardrails_test_data_names( + main_model_prefix=main_model_prefix, + workspace=workspace, + ) + topic_control_model = make_served_model( + test_id=base_test_data_names.test_id, + prefix="tc-model", + workspace=workspace, + ) return TopicControlTestDataNames( main_model_served_name=base_test_data_names.main_model_served_name, @@ -105,17 +114,17 @@ def _unsafe_input_topic_control_response() -> str: return "off-topic" @staticmethod - def _middleware_call(config_name: str) -> MiddlewareCallParam: + def _middleware_call(workspace: str, config_name: str) -> MiddlewareCallParam: return { "name": GUARDRAILS_PLUGIN_NAME, "config_type": GUARDRAILS_PLUGIN_CONFIG_TYPE, - "config_id": f"{DEFAULT_WORKSPACE}/{config_name}", + "config_id": f"{workspace}/{config_name}", } @staticmethod def _delete_config_if_present(harness: IGWLoopbackHarness, config_name: str) -> None: try: - harness.sdk.guardrail.configs.delete(name=config_name, workspace=DEFAULT_WORKSPACE) + harness.sdk.guardrail.configs.delete(name=config_name, workspace=harness.workspace) except nemo_platform.NotFoundError: pass @@ -153,10 +162,10 @@ def _config_data( return {"models": models, "rails": rails, "prompts": prompts} def _setup_entity_backed_vm(self, harness: IGWLoopbackHarness) -> TopicControlTestDataNames: - test_data_names = _make_test_data_names() + test_data_names = _make_test_data_names(workspace=harness.workspace) harness.add_provider( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.model_provider_name, served_models={ test_data_names.main_model_served_name: test_data_names.main_model_served_name, @@ -164,7 +173,7 @@ def _setup_entity_backed_vm(self, harness: IGWLoopbackHarness) -> TopicControlTe }, ) harness.sdk.guardrail.configs.create( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.guardrail_config_name, description="Entity-backed topic-control config for integration tests", data=self._config_data( @@ -174,15 +183,15 @@ def _setup_entity_backed_vm(self, harness: IGWLoopbackHarness) -> TopicControlTe ), ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.main_model_served_name, default_model_entity=test_data_names.main_model_entity_ref, ) harness.add_virtual_model( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, name=test_data_names.request_virtual_model_name, default_model_entity=test_data_names.main_model_entity_ref, - request_middleware=[self._middleware_call(test_data_names.guardrail_config_name)], + request_middleware=[self._middleware_call(harness.workspace, test_data_names.guardrail_config_name)], ) return test_data_names @@ -208,9 +217,9 @@ def _assert_inference_uses_config( ) response = harness.chat_completions( - workspace=DEFAULT_WORKSPACE, + workspace=harness.workspace, body={ - "model": f"{DEFAULT_WORKSPACE}/{test_data_names.request_virtual_model_name}", + "model": f"{harness.workspace}/{test_data_names.request_virtual_model_name}", "messages": [{"role": "user", "content": user_input}], }, ) @@ -222,7 +231,7 @@ def _assert_inference_uses_config( ) harness.assert_request_messages_contain(test_data_names.topic_control_entity_ref, user_input) guardrails_data: dict[str, Any] = response.get("guardrails_data") or {} - assert guardrails_data.get("config_ids") == [f"{DEFAULT_WORKSPACE}/{test_data_names.guardrail_config_name}"] + assert guardrails_data.get("config_ids") == [f"{harness.workspace}/{test_data_names.guardrail_config_name}"] if expect_blocked: harness.assert_no_calls_to(test_data_names.main_model_served_name) @@ -249,7 +258,7 @@ def test_input_rail( expect_blocked: bool, ) -> None: """Input topic-control rails should block off-topic user messages before they reach the backend.""" - harness = igw_loopback_harness(GuardrailsService) + harness = igw_loopback_harness() user_input = self.USER_INPUT_OFF_TOPIC if expect_blocked else self.USER_INPUT_ON_TOPIC topic_control_response = ( diff --git a/plugins/nemo-guardrails/tests/integration/utils.py b/plugins/nemo-guardrails/tests/integration/utils.py index bfd19c4b45..44bf501949 100644 --- a/plugins/nemo-guardrails/tests/integration/utils.py +++ b/plugins/nemo-guardrails/tests/integration/utils.py @@ -15,6 +15,12 @@ from nmp.testing.utils import short_unique_name DEFAULT_WORKSPACE = "default" +"""Default workspace seeded by the module-scoped IGW fixture. Helpers +default to this so parametrise-time builders that don't have a harness +yet keep working; production test bodies pass ``harness.workspace`` +explicitly so the helpers stay portable if the fixture ever moves to +per-test workspaces.""" + GUARDRAILS_PLUGIN_NAME = "nemo-guardrails" @@ -44,9 +50,19 @@ def make_served_model( return ServedModel(served_name=served_name, entity_ref=f"{workspace}/{served_name}") -def make_guardrails_test_data_names(*, main_model_prefix: str = "main-model") -> GuardrailsTestDataNames: +def make_guardrails_test_data_names( + *, + main_model_prefix: str = "main-model", + workspace: str = DEFAULT_WORKSPACE, +) -> GuardrailsTestDataNames: + """Build a unique set of names + entity refs for one test. + + Pass ``workspace=harness.workspace`` so ``main_model_entity_ref`` + lines up with the harness's workspace. The default is a safety + net for parametrise-time callers without a harness in scope. + """ test_id = short_unique_name("test") - main_model = make_served_model(test_id=test_id, prefix=main_model_prefix) + main_model = make_served_model(test_id=test_id, prefix=main_model_prefix, workspace=workspace) return GuardrailsTestDataNames( test_id=test_id, main_model_served_name=main_model.served_name, diff --git a/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/README.md b/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/README.md index 1c28471e22..6f2e2502c1 100644 --- a/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/README.md +++ b/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/README.md @@ -8,15 +8,43 @@ code (`process_request`, `process_response`, `process_post_response`) runs with its production implementation — the only mock is the upstream model provider itself, which every offline test needs. +The heavy ASGI stack (FastAPI app, SQLite-backed entity store, dependency +wiring, `/health/ready` polling, workspace seeding) is built **once per +test file** and shared across every test in the module. Per-test concerns +(mock-NIM handler mount, post-response list reset, entity teardown) still +run per test. See [Module scope and xdist](#module-scope-and-xdist) for +the resulting pytest command-line constraints. + ## Quick start -### 1. Re-export the fixture in your plugin's `conftest.py` +### 1. Re-export the fixtures in your plugin's `conftest.py` ```python # plugins//tests/integration/conftest.py -from nmp.core.inference_gateway.testing.fixtures import igw_plugin_harness +from nmp.core.inference_gateway.testing.fixtures import ( + _igw_app_context, + _igw_extra_services, + igw_plugin_harness, +) + +__all__ = ["_igw_app_context", "_igw_extra_services", "igw_plugin_harness"] +``` + +`_igw_app_context` and `_igw_extra_services` are module-scoped +fixtures that `igw_plugin_harness` depends on; pytest needs them in +the same conftest scope to resolve the dependency chain. Add +`_igw_loopback_context` too if you use `igw_loopback_harness`. -__all__ = ["igw_plugin_harness"] +#### Mounting extra services + +Override `_igw_extra_services` in your conftest to mount additional +services on the module app (e.g. `GuardrailsService` for entity-backed +guardrail-config tests): + +```python +@pytest.fixture(scope="module") +def _igw_extra_services() -> tuple[ServiceFactory, ...]: + return (GuardrailsService,) ``` ### 2. Write a test @@ -57,8 +85,8 @@ def test_safe_input_reaches_backend(igw_plugin_harness: IGWPluginHarness) -> Non | Fixture | When to use | |---|---| -| `igw_plugin_harness` | Default. No real port for IGW; plugin outbound HTTP goes directly to the mock NIM via `nim_base_url`. | -| `igw_loopback_harness` | Factory for tests where plugin outbound HTTP needs to traverse IGW (e.g. the plugin calls `get_openai_compatible_inference_url_and_model` and the resulting URL must be reachable). Call `h = igw_loopback_harness()`, which includes IGW + Models by default, or pass extra service classes like `igw_loopback_harness(GuardrailsService)` to mount additional routes. Costs a uvicorn thread + per-request `aiohttp.ClientSession` override. | +| `igw_plugin_harness` | Default. No real port for IGW; plugin outbound HTTP goes straight to the mock NIM via `nim_base_url`. | +| `igw_loopback_harness` | Factory for tests where plugin outbound HTTP needs to traverse IGW (e.g. the plugin calls `get_openai_compatible_inference_url_and_model` and the returned URL must be reachable). Call as `h = igw_loopback_harness()` — passing extra services raises `TypeError`; use `_igw_extra_services` instead. Costs a (module-scoped) uvicorn thread plus a per-test `aiohttp.ClientSession` override (scoped to loopback tests only, so plain `igw_plugin_harness` tests in the same module aren't affected). | ### 4. Choose a plugin registration method @@ -92,7 +120,7 @@ These run the same code as production: 4. **`get_platform_config()`** — patched in the loopback variant so the resolver returns the loopback URL. 5. **`global_http_client`** — replaced with per-request sessions in the loopback variant (loop-binding workaround). 6. **Passthrough VM auto-creation** — the `provider_reconciler` doesn't run. Tests needing the resolver must create passthrough VMs manually. -7. **Background cache-refresh task** — tests refresh synchronously inside `add_provider` / `add_virtual_model`. +7. **Background cache-refresh task** — disabled (`refresh_model_cache_interval_sec=0`) so the 3-second loop can't fire between tests in a module and re-populate the cache with stale rows. Tests refresh synchronously inside `add_provider` / `add_virtual_model`. 8. **Authorization** — disabled by default (`auth_enabled=False`). ## What cannot be tested @@ -109,12 +137,20 @@ These run the same code as production: | Method | Description | |---|---| -| `add_provider(workspace, served_models, ...)` | Register a `ModelProvider` routed at the mock NIM. Call **before** `add_virtual_model`. | -| `add_virtual_model(workspace, name, ...)` | Create a `VirtualModel` and refresh caches so it routes immediately. | +| `add_provider(workspace, served_models, ...)` | Register a `ModelProvider` routed at the mock NIM. Call **before** `add_virtual_model`. Tracked for entity-store cleanup. | +| `add_virtual_model(workspace, name, ...)` | Create a `VirtualModel` and refresh caches so it routes immediately. Tracked for entity-store cleanup. | +| `create_secret(workspace, name, value, ...)` | Create a Secret via the SDK and track it for harness cleanup. Use this instead of `harness.sdk.secrets.create(...)` so the secret is deleted between tests in a module-scoped fixture. | | `mock_chat_completions(model, responses)` | Queue mock responses for a model. Responses are consumed in order; the last is reused if drained. | | `load_plugin(name)` / `use_plugin(name, instance)` | Register a plugin (context manager). | | `refresh_caches()` | Full model + VM cache refresh. Needed when `api_key_secret_name` is set on a provider. | +### Workspace + +`harness.workspace` — the workspace the module-scoped fixture seeded +(`"default"` today). Use this instead of hardcoding `"default"` in +test bodies; the day the harness moves to per-test workspaces, only +the fixture changes. + ### Inference | Method | Description | @@ -155,6 +191,89 @@ Defined in `nmp.testing.mock_chat_completions`: | `chat_completion(content, model, ...)` | Builder for a non-streaming response body. | | `chat_completion_chunk(content, model, ...)` | Builder for a single SSE chunk body. | +## Module scope and xdist + +The expensive ASGI stack is wrapped in the module-scoped +`_igw_app_context`. Without that, every parametrised test pays the +full ~3–10s build cost; with it, the build amortises across the file. + +**xdist requirement.** Only `--dist=loadfile` (one file per worker) and +`--dist=loadscope` (one fixture scope per worker) preserve module +scope. The default `--dist=load` distributes individual tests across +workers, so each worker rebuilds the app from scratch and defeats the +optimisation. Run integration tests with: + +```bash +uv run --frozen pytest plugins//tests/integration --dist=loadfile -n auto +``` + +`loadfile` is the safer default — it also keeps every test in a file +on the same worker, matching the fixture lifecycle exactly. + +## Entity teardown across tests + +The harness tracks every entity it creates and deletes them on +teardown in FK order: virtual models → providers → secrets. Each +delete is guarded so one failure can't mask the test's real failure. +After deletion the in-memory caches are rebuilt so the next test's +`add_provider` doesn't see ghost `ModelProviderInfo` rows. + +**Only entities created through the harness are tracked.** A direct +`harness.sdk..create(...)` call leaks across tests under +module scope, and may then be picked up by the next +`refresh_model_cache` — triggering `notify_upserted` on a dead VM, or +making `add_provider` see stale provider rows. + +For secrets, use `harness.create_secret(...)`. For other entity types +you need to create outside the harness, either append to the relevant +tracking list yourself (e.g. `harness._secrets.append((ws, name))`) or +delete in an explicit `try/finally` around the test body. + +## Plugin lifecycle and shared SDK clients + +`use_plugin` / `load_plugin` run the plugin's `on_startup` on enter +and `on_shutdown` on exit. The catch with module scope: the shared +SDK HTTP client now lives across tests, so a plugin's `on_shutdown` +calling `await sdk.close()` (as `nemo-guardrails` does) would close +the shared client and break every later test in the module. + +The module fixture monkey-patches the shared client's `aclose` to a +no-op for the module's lifetime. `ASGITransport` is in-process so +nothing actually leaks. Plugin authors don't need to do anything +special — `on_shutdown` still runs; only the close is intercepted. + +If your plugin owns separate resources (custom pools, background +tasks, on-disk caches), close those normally — only the shared SDK's +close is intercepted. + +## Limitations under module scope + +A few patterns that worked under function scope quietly break under +module scope: + +* **Function-scoped autouse `monkeypatch` fixtures that need to + affect service startup.** `_igw_app_context` builds the app — + including every service's `on_startup` — before any function-scoped + fixture runs, so `monkeypatch.setenv` in a per-test autouse fixture + lands too late. Set the value at conftest import time + (`os.environ.setdefault(...)` at module level) or in a + `scope="module", autouse=True` fixture. The `nemo-guardrails` + conftest's `HF_HUB_OFFLINE` setup is the worked example. +* **Direct `harness.sdk..create(...)` calls** — leaks; see + [Entity teardown](#entity-teardown-across-tests). +* **Per-call extra services to `igw_loopback_harness`** — now raises + `TypeError`. Override `_igw_extra_services` instead. +* **Entry-point-registered plugins won't get + `on_virtual_model_destroyed`** on teardown — only `registry.evict` + runs, so any per-VM state the plugin tracks leaks across tests in + the module. Register such plugins per-test via + `harness.use_plugin` / `harness.load_plugin` so the plugin instance + is discarded with the test. +* **Class-level state on plugins under `load_plugin`** is shared + across the module. `load_plugin` builds a fresh instance per test, + so instance state is fine — keep caches on the instance, not on + the class. + ## File layout ``` diff --git a/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/fixtures.py b/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/fixtures.py index daccc6f184..3c6cda1236 100644 --- a/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/fixtures.py +++ b/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/fixtures.py @@ -3,11 +3,30 @@ """Pytest fixtures for the IGW middleware test harnesses. -Importing :func:`igw_plugin_harness` (or :func:`igw_loopback_harness`) -into a test module — or re-exporting from a project ``conftest.py`` — -registers the fixture for the surrounding scope. Both piggyback on -``pytest_httpserver``'s function-scoped ``httpserver`` fixture so each -test gets an isolated socket and clean handler state. +Import :func:`igw_plugin_harness` or :func:`igw_loopback_harness` into a +test module (or re-export from a project ``conftest.py``) to register +the fixture. Both use ``pytest_httpserver``'s function-scoped +``httpserver`` so each test gets a fresh mock-NIM socket. + +The ASGI stack is split across two scopes so each test only pays for +what changes between tests: + +* :func:`_igw_app_context` (**module**) — the heavy + ``create_test_client`` call (SQLite DB, FastAPI app, IGW + Models + services, ``/health/ready`` polling, workspace seeding). The periodic + ``refresh_model_cache_task`` is disabled so it can't wake mid-test + and re-pollute the cache. +* :func:`_igw_loopback_context` (**module**) — a uvicorn thread bound + on top of the app context. Only entered when a test asks for the + loopback variant. +* :func:`igw_plugin_harness` / :func:`igw_loopback_harness` + (**function**) — a fresh :class:`IGWPluginHarness` per test: mock-NIM + handler mount, post-response task list re-init, entity teardown. + +**xdist**: only ``--dist=loadfile`` and ``--dist=loadscope`` preserve +module scope. The default ``--dist=load`` distributes individual tests +across workers and defeats the speed-up. See the README for the +recommended command line. """ from collections.abc import Callable, Generator @@ -21,139 +40,224 @@ from pytest_httpserver import HTTPServer -def _enable_post_response_task_tracking(client_context: ClientContext) -> None: - """Initialise ``app.state.pending_post_response_tasks`` so ``proxy.py`` records them. +def _app_from(client_context: ClientContext) -> FastAPI: + """``TestClient.app`` is typed as bare ``ASGIApp``; we need :class:`FastAPI`. - ``proxy.py`` checks for this attribute on every request that schedules a - fire-and-forget post-response task; production never sets it, so the - list-or-None guard keeps the production hot path free of test-only - state. Only the test harness initialises it here, and only the harness's - :meth:`IGWPluginHarness.aflush_post_response` reads it. - """ - # ``TestClient.app`` is typed as a bare ``ASGIApp`` callable but is in - # fact our :class:`FastAPI` instance — narrow the type so ``state`` is - # accessible. - app = cast(FastAPI, client_context.test_client.app) - app.state.pending_post_response_tasks = [] - - -def _register_global_state_resets(stack: ExitStack) -> None: - """Register IGW global-state resets so they run on fixture teardown. - - The harness's ``_cleanup`` evicts its own VMs and restores its own plugin - registrations, but a partial cleanup (e.g. an exception in ``_build`` - before the harness is fully wired) could leave global state populated. - Resetting all three globals on teardown guarantees the next test starts - with empty caches and an empty registry, regardless of whether ``_cleanup`` - ran successfully. - - Each reset is independent (``_GLOBAL = None``), so execution order is - irrelevant. Do not pair this with ``igw_mock_provider_mode=True`` in - ``create_test_client`` — that mode already registers its own - ``reset_global_model_cache`` callback and a double-reset is confusing - in a debugger even though it's idempotent. + Centralised so the cast is justified in one place. """ - from nmp.core.inference_gateway.api.dependencies import ( - reset_global_middleware_registry, - reset_global_model_cache, - reset_global_virtual_model_cache, - ) + return cast(FastAPI, client_context.test_client.app) + + +def _enable_post_response_task_tracking(client_context: ClientContext) -> None: + """Reset the per-test list ``proxy.py`` appends fire-and-forget tasks to. - stack.callback(reset_global_middleware_registry) - stack.callback(reset_global_virtual_model_cache) - stack.callback(reset_global_model_cache) + Production leaves ``app.state.pending_post_response_tasks`` unset and + ``proxy.py`` skips tracking; only the harness sets it (read by + :meth:`IGWPluginHarness.aflush_post_response`). Reset per test so a + stale list from the previous test can't pin completed tasks or get + re-awaited. + """ + _app_from(client_context).state.pending_post_response_tasks = [] -def _create_harness_client_context( - stack: ExitStack, +@contextmanager +def _build_app_context( *extra_services: ServiceFactory, -) -> ClientContext: - """Create the shared in-process IGW + Models client context for harness fixtures.""" - # Local imports keep this module cheap to import for unrelated test files. +) -> Generator[ClientContext, None, None]: + """Yield an IGW + Models + extras :class:`ClientContext` (module-lived). + + Two module-scope hazards are neutralised here: + + 1. The 3-second background ``refresh_model_cache_task``. ``on_startup`` + reads ``refresh_model_cache_interval_sec`` from the module-level + config snapshot (captured at first import), so a ``service_configs`` + override is too late. Patch the snapshot field to 0 *before* + entering ``create_test_client`` and ``on_startup`` never schedules + the loop. + 2. The shared SDK HTTP client's ``aclose``. Plugins like + ``nemo-guardrails`` call ``await sdk.close()`` in ``on_shutdown``, + which would close the shared client for every later test in the + module. Patch ``aclose`` to a no-op for the module's lifetime; + ``ASGITransport`` is in-process so nothing actually leaks. + """ + from unittest.mock import patch + + from nmp.common import sdk_factory as sdk_factory_module + from nmp.core.inference_gateway import config as igw_config_module from nmp.core.inference_gateway.service import InferenceGatewayService from nmp.core.models.service import ModelsService service_types: list[ServiceFactory] = [InferenceGatewayService, ModelsService, *extra_services] - _register_global_state_resets(stack) - - client_context = stack.enter_context( - create_test_client( + with patch.object(igw_config_module.config, "refresh_model_cache_interval_sec", 0): + with create_test_client( *service_types, client_type=ClientContext, igw_mock_provider_mode=False, - ) - ) - _enable_post_response_task_tracking(client_context) + ) as client_context: + shared_async_client = sdk_factory_module._test_http_client + if shared_async_client is None: + yield client_context + return + + original_aclose = shared_async_client.aclose + + async def _noop_aclose() -> None: + return None + + shared_async_client.aclose = _noop_aclose # type: ignore[method-assign] + try: + yield client_context + finally: + shared_async_client.aclose = original_aclose # type: ignore[method-assign] + + +@pytest.fixture(scope="module") +def _igw_extra_services() -> tuple[ServiceFactory, ...]: + """Override in a plugin conftest to mount extra services module-wide. + + Example:: + + @pytest.fixture(scope="module") + def _igw_extra_services() -> tuple[ServiceFactory, ...]: + from nmp.guardrails.service import GuardrailsService + + return (GuardrailsService,) + + Module-scoped because the app it feeds is module-scoped. + """ + return () - return client_context + +@pytest.fixture(scope="module") +def _igw_app_context( + _igw_extra_services: tuple[ServiceFactory, ...], +) -> Generator[ClientContext, None, None]: + """Module-scoped IGW + Models ASGI stack. + + The expensive ``create_test_client`` call runs once per test file; + function-scoped fixtures layer per-test concerns on top. Extra + services come from :func:`_igw_extra_services` rather than fixture + parameters so plugin conftests can declare their needs without + rebuilding the app per-test. + """ + with _build_app_context(*_igw_extra_services) as client_context: + yield client_context + + +@pytest.fixture(scope="module") +def _igw_loopback_context( + _igw_app_context: ClientContext, +) -> Generator[str, None, None]: + """Run the module app on a real ``127.0.0.1:`` and yield its URL. + + Only entered when a test actually requests :func:`igw_loopback_harness` + — plain modules pay nothing for uvicorn. + + The per-request HTTP client override and the ``get_platform_config`` + patch live in :func:`_build_loopback_harness` instead, so plain + ``igw_plugin_harness`` tests in a mixed module don't pay loopback's + per-request session cost. Uvicorn is started with ``lifespan="off"``; + the TestClient still owns startup/shutdown. + """ + from nmp.core.inference_gateway.testing._loopback import serve_app_in_thread + + with serve_app_in_thread(_app_from(_igw_app_context)) as loopback_base_url: + yield loopback_base_url @contextmanager -def _igw_plugin_harness_context( +def _per_test_plugin_setup( + client_context: ClientContext, httpserver: HTTPServer, - *extra_services: ServiceFactory, ) -> Generator[IGWPluginHarness, None, None]: - """IGW + Models in-process via ASGI; mock NIM via ``pytest_httpserver``. + """Per-test setup/teardown shared by the plain + loopback fixtures. - No uvicorn thread, no real port for IGW. The same ``pytest_httpserver`` - socket serves both the proxy step's outbound HTTP and any plugin-side - outbound HTTP (e.g. Guardrails' rail calls). + Resets the post-response task list, builds a fresh harness on the + per-test ``pytest_httpserver`` socket, and runs + :meth:`IGWPluginHarness._cleanup` on teardown to delete this test's + entities and rebuild the in-memory caches. - Includes IGW + Models by default. Pass additional service classes when a - test needs their routes mounted in the same app. + Note we deliberately don't call the original ``reset_global_*`` + helpers between tests — the module-scoped app's ``on_startup`` is + the only thing that re-initialises those globals, so nulling them + would crash the next request. """ - with ExitStack() as stack: - client_context = _create_harness_client_context(stack, *extra_services) - harness = IGWPluginHarness._build(client_context=client_context, mock_nim=httpserver) - stack.callback(harness._cleanup) + _enable_post_response_task_tracking(client_context) + harness = IGWPluginHarness._build(client_context=client_context, mock_nim=httpserver) + try: yield harness + finally: + harness._cleanup() @pytest.fixture -def igw_plugin_harness(httpserver: HTTPServer) -> Generator[IGWPluginHarness, None, None]: - with _igw_plugin_harness_context(httpserver) as harness: +def igw_plugin_harness( + _igw_app_context: ClientContext, + httpserver: HTTPServer, +) -> Generator[IGWPluginHarness, None, None]: + """Per-test IGW + Models harness — no real port, mock NIM only. + + Cheap: the heavy ASGI stack comes from the module-scoped + :func:`_igw_app_context`. Per test you only pay for the harness + construction, the post-response list reset, and the function-scoped + mock-NIM socket. + """ + with _per_test_plugin_setup(_igw_app_context, httpserver) as harness: yield harness @contextmanager -def _igw_loopback_harness_context( +def _build_loopback_harness( + client_context: ClientContext, httpserver: HTTPServer, + igw_loopback_base_url: str, *extra_services: ServiceFactory, ) -> Generator[IGWLoopbackHarness, None, None]: - """IGW + Models in-process *and* reachable on a real ``127.0.0.1:``. - - Includes IGW + Models by default. Pass additional service classes when a - test needs their routes mounted in the same app. - - See :class:`IGWLoopbackHarness` for the two-loop loop-binding caveat. - - Three things the loopback shape requires that the default doesn't: - - 1. Lifecycle ordering: ASGI client owns the app's startup/shutdown, - uvicorn comes up after the app is ready and tears down before it. - 2. Per-request ``aiohttp.ClientSession`` override — the app now runs on - two loops (``TestClient``'s and uvicorn's). Sessions are loop-bound, - so a singleton would fail with "attached to a different loop" on - whichever loop didn't originate it. Production is unaffected (one - loop per process). - 3. ``platform_config.base_url`` patched to the loopback URL so the - plugin resolver - (:meth:`get_openai_compatible_inference_url_and_model`) returns URLs - reachable from the test process instead of the production default - ``http://localhost:8080``. + """Per-test setup/teardown for the loopback harness. + + Same per-test resets as :func:`_per_test_plugin_setup`, plus two + loopback-only patches scoped to this test: + + * ``per_request_http_client`` overrides :func:`global_http_client` — + loopback runs the app from two loops (TestClient + uvicorn) and + a singleton :class:`aiohttp.ClientSession` would be loop-bound to + whichever one created it. + * ``get_platform_config`` is patched at IGW's middleware-registry + import site so :meth:`get_openai_compatible_inference_url_and_model` + returns URLs reachable from the test process. + + Both patches roll back before the next test runs, so a plain + ``igw_plugin_harness`` test sharing the module doesn't observe them. + + Raises: + TypeError: If *extra_services* is non-empty. The module-scoped + app is already built by the time this runs; the previous + ``igw_loopback_harness(GuardrailsService)`` pattern is dead. + Override :func:`_igw_extra_services` in your conftest + instead. (Hard error, not a warning, because pytest hides + ``DeprecationWarning`` by default and silently-missing + routes would be much harder to diagnose.) """ + if extra_services: + names = ", ".join(getattr(s, "__name__", repr(s)) for s in extra_services) + raise TypeError( + f"igw_loopback_harness({names}): extra service args are no longer " + "accepted under module-scoped fixtures. Override " + "`_igw_extra_services` in your conftest to mount additional " + "services for the whole module." + ) from nmp.core.inference_gateway.api.dependencies import global_http_client from nmp.core.inference_gateway.testing._loopback import ( override_platform_base_url, per_request_http_client, - serve_app_in_thread, ) - with ExitStack() as stack: - client_context = _create_harness_client_context(stack, *extra_services) + _enable_post_response_task_tracking(client_context) + + app = _app_from(client_context) - app = cast(FastAPI, client_context.test_client.app) + with ExitStack() as stack: previous_override = app.dependency_overrides.get(global_http_client) app.dependency_overrides[global_http_client] = per_request_http_client @@ -164,40 +268,53 @@ def _restore_http_client_override() -> None: app.dependency_overrides[global_http_client] = previous_override stack.callback(_restore_http_client_override) + stack.enter_context(override_platform_base_url(igw_loopback_base_url)) - loopback_base_url = stack.enter_context(serve_app_in_thread(app)) - # Patch goes below uvicorn so it tears down first. - stack.enter_context(override_platform_base_url(loopback_base_url)) harness = cast( IGWLoopbackHarness, IGWLoopbackHarness._build( client_context=client_context, mock_nim=httpserver, - igw_loopback_base_url=loopback_base_url, + igw_loopback_base_url=igw_loopback_base_url, ), ) - stack.callback(harness._cleanup) - yield harness + try: + yield harness + finally: + harness._cleanup() @pytest.fixture def igw_loopback_harness( + _igw_app_context: ClientContext, + _igw_loopback_context: str, httpserver: HTTPServer, ) -> Generator[Callable[..., IGWLoopbackHarness], None, None]: - """Factory for an IGW loopback harness. - - Call with no arguments for the default IGW + Models app: - ``harness = igw_loopback_harness()``. + """Factory for an IGW loopback harness — call ``igw_loopback_harness()``. - Pass additional service classes to mount them in the same app: - ``harness = igw_loopback_harness(GuardrailsService)``. + Passing extra service classes raises :class:`TypeError`. Mount extra + services by overriding :func:`_igw_extra_services` in the plugin's + conftest. """ with ExitStack() as stack: def factory(*extra_services: ServiceFactory) -> IGWLoopbackHarness: - return stack.enter_context(_igw_loopback_harness_context(httpserver, *extra_services)) + return stack.enter_context( + _build_loopback_harness( + _igw_app_context, + httpserver, + _igw_loopback_context, + *extra_services, + ) + ) yield factory -__all__ = ["igw_plugin_harness", "igw_loopback_harness"] +__all__ = [ + "_igw_app_context", + "_igw_extra_services", + "_igw_loopback_context", + "igw_loopback_harness", + "igw_plugin_harness", +] diff --git a/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/harness.py b/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/harness.py index 108b0f5f74..e60e188c9b 100644 --- a/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/harness.py +++ b/services/core/inference-gateway/src/nmp/core/inference_gateway/testing/harness.py @@ -5,22 +5,21 @@ Design: -- One real HTTP boundary owned by ``pytest_httpserver``. The IGW + Models - app runs in-process via ``httpx.ASGITransport`` (no uvicorn, no real - port). Both the proxy step's outbound HTTP and any plugin-side outbound - HTTP terminate at the same socket. -- Providers are plain (not ``igw-mock-`` prefixed) — their ``host_url`` - points at the mock NIM so IGW issues real HTTP that the mock answers. +- ``pytest_httpserver`` owns the only real socket. The IGW + Models app + runs in-process via ``httpx.ASGITransport``. Both the proxy step's + outbound HTTP and any plugin-side outbound HTTP terminate at the same + mock-NIM socket. +- Providers are plain (no ``igw-mock-`` prefix); their ``host_url`` + points at the mock NIM so IGW issues real HTTP. - Assertions read the mock NIM's per-call request log rather than - matching response IDs through the proxy. + threading response IDs through the proxy. -Sync entry points (:meth:`IGWPluginHarness.add_virtual_model`, etc.) call -``asyncio.run`` internally, so they must not run inside a live event -loop. Async tests use the ``a``-prefixed siblings -(:meth:`aadd_virtual_model`, :meth:`achat_completions`, -:meth:`ause_plugin`). +Sync entry points (:meth:`IGWPluginHarness.add_virtual_model`, etc.) +call :func:`asyncio.run` internally and must not run inside a live loop. +Use the ``a``-prefixed siblings (:meth:`aadd_virtual_model`, +:meth:`achat_completions`, :meth:`ause_plugin`) from async tests. -Companion fixture: :mod:`nmp.core.inference_gateway.testing.fixtures`. +Companion fixtures: :mod:`nmp.core.inference_gateway.testing.fixtures`. """ import asyncio @@ -68,6 +67,13 @@ DEFAULT_MOCK_CHAT_PATH = "/v1/chat/completions" +DEFAULT_WORKSPACE = "default" +"""Workspace seeded by :func:`~nmp.testing.client.create_test_client` at +module-fixture setup. Exposed on the harness as +:attr:`IGWPluginHarness.workspace` so test bodies don't hardcode +``"default"`` and stay portable if the fixture later issues a per-test +workspace.""" + @dataclass class IGWPluginHarness: @@ -75,12 +81,11 @@ class IGWPluginHarness: Owns a :class:`~nmp.testing.client.ClientContext` (sync + async SDK, :class:`TestClient`, :class:`EntityClient`) backed by an in-process - ASGI IGW + Models app, a :class:`HTTPServer` (``mock_nim``) — the - only real listening socket — and a - :class:`MockChatCompletionsHandler` pre-mounted at - ``POST /v1/chat/completions`` on it. + ASGI IGW + Models app, the mock-NIM :class:`HTTPServer` (the only + real socket), and a :class:`MockChatCompletionsHandler` pre-mounted + at ``POST /v1/chat/completions``. - Construct via :func:`igw_plugin_harness` (the pytest fixture). + Construct via the :func:`igw_plugin_harness` pytest fixture. """ sdk: NeMoPlatform @@ -89,26 +94,36 @@ class IGWPluginHarness: entity_client: EntityClient mock_nim: HTTPServer - """The single real socket in the test process. Tests can register - extra routes on it (e.g. ``/v1/embeddings``) beyond the auto-mounted + """The only real socket in the test process. Tests can register extra + routes (e.g. ``/v1/embeddings``) on top of the auto-mounted chat-completions handler. The auto-mounted handler uses ``expect_request`` (a *permanent* - matcher). A test mounting a oneshot matcher for - ``/v1/chat/completions`` wins for the first call; subsequent calls - fall back to the permanent handler — usually what tests want.""" + matcher). A oneshot matcher for ``/v1/chat/completions`` wins the + first call; subsequent calls fall through to the permanent + handler — usually what tests want.""" handler: MockChatCompletionsHandler - """Mounted chat-completions handler. Tests register responses via - :meth:`mock_chat_completions` and assert via :meth:`assert_call_count` - and friends rather than touching this directly.""" + """The auto-mounted chat-completions handler. Tests interact via + :meth:`mock_chat_completions` and :meth:`assert_call_count` rather + than touching this directly.""" + + workspace: str + """Workspace the module-scoped fixture seeded. Reach for this in test + bodies instead of a literal ``"default"``.""" _registry: MiddlewareRegistry _model_cache: ModelCache _vm_cache: VirtualModelCache _cache_accessor: InferenceMiddlewareCacheAccessorImpl _virtual_models: list[tuple[str, str]] - """``(workspace, name)`` of VMs created by this harness — torn down on cleanup.""" + """VMs created by this harness, deleted on teardown.""" + _providers: list[tuple[str, str]] + """Providers created via :meth:`add_provider`, deleted on teardown so + they can't re-enter the model cache on the next test.""" + _secrets: list[tuple[str, str]] + """Secrets created via :meth:`create_secret`, deleted on teardown + after providers.""" # ------------------------------------------------------------------ # Builder @@ -120,13 +135,13 @@ def _build( *, client_context: ClientContext, mock_nim: HTTPServer, + workspace: str = DEFAULT_WORKSPACE, **extra_fields: Any, ) -> "IGWPluginHarness": - """Construct a harness around an already-running IGW + Models app. + """Build a harness around an already-running app. - Subclasses (e.g. :class:`IGWLoopbackHarness`) pass their extra - dataclass fields through *extra_fields* so the same ``cls(...)`` - call wires up parent + subclass fields together. + Subclasses pass their extra dataclass fields via *extra_fields* + so one ``cls(...)`` call wires parent + subclass together. """ registry = global_middleware_registry() model_cache = global_model_cache() @@ -147,25 +162,93 @@ def _build( entity_client=client_context.entity_client, mock_nim=mock_nim, handler=handler, + workspace=workspace, _registry=registry, _model_cache=model_cache, _vm_cache=vm_cache, _cache_accessor=cache_accessor, _virtual_models=[], + _providers=[], + _secrets=[], **extra_fields, ) def _cleanup(self) -> None: - """Evict VMs created by this harness from runtime state.""" + """Delete this test's entities, then rebuild the in-memory caches. + + Deletes in FK order: VMs → providers → secrets. Each step + catches and logs broadly: cleanup runs in ``finally`` after the + test has already finished, and the goal is to drain every + tracked entity even if one delete raises. Narrowing to + ``APIError`` would let a programming error in cleanup code + strand the remaining deletes — the precise pollution we set + this up to prevent. Failures are logged with ``exc_info=True`` + so nothing is silently lost. + + **What this does NOT clean up:** + + * Plugins registered persistently at app startup (via the + ``nemo.inference_middleware`` entry-point group) won't see + ``on_virtual_model_destroyed`` — only ``registry.evict`` runs. + Per-VM state in such plugins leaks across tests; register them + per-test via :meth:`use_plugin` / :meth:`load_plugin` if you + need clean state. + * ``registry.broken_vms`` and + :attr:`VirtualModelCache.config_ref_versions` aren't pruned + here — they self-heal on the next + :func:`refresh_virtual_model_cache`. In practice every test + calls :meth:`add_virtual_model` which triggers a refresh. + """ + for workspace, name in reversed(self._virtual_models): + try: + self.sdk.inference.virtual_models.delete(name=name, workspace=workspace) + except Exception: # noqa: BLE001 # see _cleanup docstring + logger.warning( + "Failed to delete VirtualModel %r in workspace %r during harness cleanup", + name, + workspace, + exc_info=True, + ) + + for workspace, name in reversed(self._providers): + try: + self.sdk.inference.providers.delete(name=name, workspace=workspace) + except Exception: # noqa: BLE001 # see _cleanup docstring + logger.warning( + "Failed to delete ModelProvider %r in workspace %r during harness cleanup", + name, + workspace, + exc_info=True, + ) + + for workspace, name in reversed(self._secrets): + try: + self.sdk.secrets.delete(name=name, workspace=workspace) + except Exception: # noqa: BLE001 # see _cleanup docstring + logger.warning( + "Failed to delete Secret %r in workspace %r during harness cleanup", + name, + workspace, + exc_info=True, + ) + + # Rebuild in-memory caches to match the post-delete entity store. for key in self._virtual_models: self._registry.evict(key) - removed = set(self._virtual_models) if removed: self._vm_cache.rebuild( [vm for vm in self._vm_cache.virtual_model_map.values() if (vm.workspace, vm.name) not in removed] ) + # Drop deleted providers so the next add_provider fast-path + # doesn't see ghost ModelProviderInfo rows. + for workspace, name in self._providers: + self._model_cache.workspace_name_provider_map.pop((workspace, name), None) + self._model_cache.rebuild_model_entity_map() + self._virtual_models.clear() + self._providers.clear() + self._secrets.clear() # ------------------------------------------------------------------ # Public conveniences @@ -173,11 +256,10 @@ def _cleanup(self) -> None: @property def nim_base_url(self) -> str: - """OpenAI-compatible base URL — pass as ``parameters.base_url``. + """OpenAI-compatible base URL (``http://host:port/v1``). - Resolves to ``http://:/v1``. Both the IGW proxy step - and an OpenAI client built from this URL hit the auto-mounted - handler. + Pass as ``parameters.base_url``. Both the IGW proxy step and an + OpenAI client built from this URL hit the auto-mounted handler. """ return self.mock_nim.url_for("/v1") @@ -200,39 +282,25 @@ def use_plugin( ) -> Generator[NemoInferenceMiddleware, None, None]: """Register *plugin* under *name*; restore the prior entry on exit. - The cache accessor is injected before yield, so plugin cache - methods (``get_inference_url_and_model``, ``get_virtual_model``, - ...) work inside the context. - - **Production parity:** prefer :meth:`load_plugin` when the plugin's - package is pip-installed in the test venv — it discovers the plugin - via the same ``nemo.inference_middleware`` entry-point group IGW - uses in production. Reach for ``use_plugin`` when the plugin isn't - installable (workspace-only, like the example plugin) or when you - need to substitute a :class:`MagicMock` / - :class:`AsyncMock`-spec'd instance. - - ``call_lifecycle=True`` (the default) runs ``on_startup`` / - ``on_shutdown`` via :func:`asyncio.run`, which spins up a **fresh - disposable event loop** for each. This matches what production does - (lifespan startup + shutdown) and is what plugins like Guardrails - require to wire up their SDK / cache; the previous - ``call_lifecycle=False`` default silently 503'd every request for - such plugins. - - Loop-bound resources (``aiohttp.ClientSession``, ``asyncio.Lock`` / - ``Queue``, long-lived Tasks) created in ``on_startup`` will be torn - down on a different loop in ``on_shutdown`` and may emit "attached - to a different loop"; if the plugin *uses* such resources during - :meth:`process_request` / :meth:`process_response`, the request - will fail with the same error because the request loop differs from - the loop ``on_startup`` ran on. Drive those tests from an - ``async def`` and use :meth:`ause_plugin` instead — both lifecycle - hooks then run on the test's own loop, matching the request loop. - - Pass ``call_lifecycle=False`` to skip the hooks entirely (useful - for ``MagicMock(spec=Plugin)`` substitutions or for tests that - manually drive the lifecycle). + The cache accessor is injected before yield so plugin cache methods + work inside the context. Prefer :meth:`load_plugin` for any + pip-installed plugin so the test exercises its entry-point + declaration; use this method for workspace-only plugins or to + substitute a :class:`MagicMock`-spec'd instance. + + With ``call_lifecycle=True`` (the default), ``on_startup`` and + ``on_shutdown`` each run via :func:`asyncio.run` — i.e. on a + fresh disposable event loop. Plugins that build loop-bound + resources in ``on_startup`` (``aiohttp.ClientSession``, + ``asyncio.Lock``, long-running Tasks) and then use them during + a request will fail with "attached to a different loop": the + request runs on yet another loop. Drive those tests from + ``async def`` and use :meth:`ause_plugin` so both hooks share the + test's own loop. + + Pass ``call_lifecycle=False`` to skip the hooks — useful for + ``MagicMock(spec=Plugin)`` or tests that drive the lifecycle + themselves. """ original_present = name in self._registry.plugins original = self._registry.plugins.get(name) @@ -249,11 +317,11 @@ def use_plugin( else: self._registry.plugins.pop(name, None) if call_lifecycle: - # Log instead of raising so cleanup failures are visible - # without masking the test outcome. + # Plugin code can raise anything; log rather than raise + # so a teardown failure doesn't mask the test outcome. try: asyncio.run(plugin.on_shutdown()) - except Exception: + except Exception: # noqa: BLE001 logger.warning( "Plugin %r on_shutdown raised during use_plugin teardown", name, @@ -270,11 +338,11 @@ async def ause_plugin( ) -> AsyncGenerator[NemoInferenceMiddleware, None]: """Async variant of :meth:`use_plugin`. - Both lifecycle hooks run on the test's own running loop (the same - loop the request will execute on), so loop-bound resources created - in ``on_startup`` stay valid through ``on_shutdown``. This is the - loop-safe alternative to :meth:`use_plugin` for plugins whose - startup builds long-lived loop-bound resources. + Both lifecycle hooks run on the test's own loop (the same loop + the request runs on), so loop-bound resources created in + ``on_startup`` stay valid through ``on_shutdown``. Use this + instead of :meth:`use_plugin` for plugins that build long-lived + loop-bound resources at startup. """ original_present = name in self._registry.plugins original = self._registry.plugins.get(name) @@ -291,9 +359,10 @@ async def ause_plugin( else: self._registry.plugins.pop(name, None) if call_lifecycle: + # See use_plugin for why this is a blind catch. try: await plugin.on_shutdown() - except Exception: + except Exception: # noqa: BLE001 logger.warning( "Plugin %r on_shutdown raised during ause_plugin teardown", name, @@ -307,33 +376,26 @@ def load_plugin( *, call_lifecycle: bool = True, ) -> Generator[NemoInferenceMiddleware, None, None]: - """Load *name* via the production ``nemo.inference_middleware`` entry-point group. - - This is the production-parity path — IGW's - :func:`~nmp.core.inference_gateway.api.middleware_registry.load_middleware_plugins` - uses the same :func:`~nemo_platform_plugin.discovery.discover_inference_middleware` - function to walk the same entry-point group. A test that goes through - ``load_plugin`` therefore exercises the entry-point declaration in - the plugin's ``pyproject.toml`` and catches misconfigurations - (missing entry-point key, wrong import path, broken class import) - that :meth:`use_plugin` silently glosses over. - - Use :meth:`use_plugin` only when: - - - The plugin isn't pip-installed in the test venv (workspace-only - plugins like the example plugin are not discoverable). - - You need to substitute a :class:`MagicMock` / - :class:`AsyncMock`-spec'd instance. - - You need to pre-configure plugin instance state before - ``on_startup`` runs. - - ``call_lifecycle`` defaults to ``True`` — same as :meth:`use_plugin`; - see that method's docstring for the loop-binding caveat. For plugins - that build loop-bound resources in ``on_startup``, drive the test - from an ``async def`` and use :meth:`aload_plugin`. + """Load *name* via the ``nemo.inference_middleware`` entry-point group. + + Production-parity: IGW's :func:`load_middleware_plugins` walks + the same entry-point group, so this exercises the plugin's + ``pyproject.toml`` declaration and catches misconfigurations + (missing key, wrong import path) that :meth:`use_plugin` would + silently gloss over. + + Use :meth:`use_plugin` instead when: + + - The plugin isn't pip-installed (workspace-only plugins). + - You need a :class:`MagicMock` / :class:`AsyncMock` instance. + - You need to pre-configure instance state before ``on_startup``. + + ``call_lifecycle`` carries the same loop-binding caveat as + :meth:`use_plugin` — use :meth:`aload_plugin` for plugins with + loop-bound startup resources. Raises: - ValueError: If no plugin is registered under *name* in the + ValueError: If *name* isn't registered in the ``nemo.inference_middleware`` entry-point group. """ instance = _instantiate_discovered_plugin(name) @@ -353,9 +415,34 @@ async def aload_plugin( yield plugin # ------------------------------------------------------------------ - # Provider / VirtualModel creation (refresh hidden) + # Secret / Provider / VirtualModel creation (refresh hidden) # ------------------------------------------------------------------ + def create_secret( + self, + *, + workspace: str, + name: str, + value: str, + description: str | None = None, + ) -> str: + """Create a Secret and track it so the harness deletes it on teardown. + + Prefer this over a direct ``self.sdk.secrets.create(...)`` — only + harness-tracked entities get cleaned up, and an untracked secret + will leak across tests under module scope (and keep a deleted + provider's ``api_key_secret_name`` alive on the next refresh). + + Returns *name* so it chains cleanly into + :meth:`add_provider` (``api_key_secret_name=harness.create_secret(...)``). + """ + kwargs: dict[str, Any] = {"workspace": workspace, "name": name, "value": value} + if description is not None: + kwargs["description"] = description + self.sdk.secrets.create(**kwargs) + self._secrets.append((workspace, name)) + return name + def add_provider( self, *, @@ -366,52 +453,39 @@ def add_provider( enabled_models: Sequence[str] | None = None, api_key_secret_name: str | None = None, ) -> ModelProvider: - """Register a (real, non-mock) ModelProvider routed at the mock NIM. - - Plain provider — no ``igw-mock-`` prefix — so the proxy step - issues a real HTTP request, terminating at ``mock_nim`` because - ``host_url`` defaults to :attr:`nim_host_url`. + """Register a real (non-mock) ModelProvider pointing at the mock NIM. Call this **before** :meth:`add_virtual_model` for any VM that - references the provider — the VM-cache refresh resolves - middleware configs against the model cache, and an unknown - ``default_model_entity`` silently produces an empty - pre-resolved-call list for that VM. - - **Auth-aware refresh:** when *api_key_secret_name* is set, the - method runs the full :func:`refresh_model_cache` after creation - so the provider's ``secret_value`` is resolved via the secrets - SDK; without that, the proxy would reject inference with HTTP - 424. Without *api_key_secret_name*, the method takes a fast path - that updates the model cache in-place without secret-resolution - plumbing. + references the provider — an unknown ``default_model_entity`` + silently produces an empty pre-resolved-call list rather than + raising. + + When *api_key_secret_name* is set, this runs the full + :func:`refresh_model_cache` so the secret value is resolved via + the secrets SDK (otherwise the proxy would 424). Without it, the + method takes a fast path that updates the cache in place. Args: workspace: Provider workspace. served_models: ``model_entity_name`` → ``served_model_name``. The served name is what arrives at the upstream — register - handler responses under the same key. - name: Provider name. Auto-generated if omitted (recommended; - fixture isolation guarantees uniqueness). An explicit - duplicate name raises ``ConflictError`` so isolation - breakage fails loudly rather than masked by delete+recreate. - host_url: Override the default mock NIM URL. + mock handler responses under the same key. + name: Provider name. Auto-generated if omitted (recommended). + An explicit duplicate raises ``ConflictError`` so isolation + breakage fails loudly. + host_url: Override the default mock-NIM URL. enabled_models: Optional enabled-models list for the SDK. - api_key_secret_name: Name of an existing platform Secret to - attach as the provider's bearer token. The secret must - already exist in *workspace* — create it via - ``self.sdk.secrets.create(...)`` before calling this - method. When set, triggers a full cache refresh so the - secret value is resolved. + api_key_secret_name: Existing Secret name to attach as the + provider's bearer token. Create it via + :meth:`create_secret` first. Triggers a full cache refresh + so the secret value is resolved. Returns: - The created ``ModelProvider`` (read back via ``retrieve`` - after ``update_status``, so its ``id`` and ``served_models`` - reflect entity-store state). + The provider, read back after ``update_status`` so its ``id`` + and ``served_models`` reflect entity-store state. Raises: - ConflictError: If a provider with *name* already exists in - *workspace*. + ConflictError: If *name* already exists in *workspace*. """ from nmp.testing.utils import short_unique_name @@ -425,9 +499,11 @@ def add_provider( enabled_models=list(enabled_models) if enabled_models is not None else omit, api_key_secret_name=api_key_secret_name if api_key_secret_name is not None else omit, ) + # Track right after create so a later raise from update_status / + # retrieve still leaves the provider eligible for teardown. + self._providers.append((workspace, provider_name)) - # served_models persisted via update_status (the create path - # doesn't accept them) so they survive future cache refreshes. + # served_models has to go through update_status — the create path doesn't accept it. self.sdk.inference.providers.update_status( name=provider_name, workspace=workspace, @@ -466,22 +542,19 @@ def add_virtual_model( response_middleware: Sequence[MiddlewareCallParam] = (), post_response_middleware: Sequence[MiddlewareCallParam] = (), ) -> SDKVirtualModel: - """Create a VirtualModel and refresh IGW's VM cache so it routes immediately. + """Create a VirtualModel and refresh the VM cache so it routes immediately. - Sync entry point — the cache refresh runs via :func:`asyncio.run`, - so this must not run inside a live event loop. Use - :meth:`aadd_virtual_model` from async tests. + Sync entry — uses :func:`asyncio.run`, so don't call this inside + a live loop. Use :meth:`aadd_virtual_model` from async tests. Call :meth:`add_provider` first for any provider this VM references; otherwise middleware-config pre-resolution sees an empty model cache. - ``models`` is the per-VM list of model entity references with - optional ``backend_format`` overrides. Plugins like - ``nemo-switchyard`` read ``virtual_model.models`` in - :meth:`on_virtual_model_upserted` to build their format-aware - routing tables; pass an entry per backend the test needs. - Format:: + ``models`` is the per-VM list of entity refs with optional + ``backend_format`` overrides. Plugins like ``nemo-switchyard`` + read this in ``on_virtual_model_upserted`` to build their + routing tables:: models=[ {"model": "default/main", "backend_format": "OPENAI_CHAT"}, @@ -489,21 +562,14 @@ def add_virtual_model( ] .. note:: - **Plugin-raised errors during VM upsert are swallowed.** Both - :meth:`NemoInferenceMiddleware.validate_middleware_config` and - :meth:`NemoInferenceMiddleware.on_virtual_model_upserted` - failures are caught by IGW's - :class:`MiddlewareRegistry` (logged-and-continued, never - re-raised). So a plugin that rejects a VM at upsert time - (e.g. switchyard's ``translate``-in-``response_middleware`` - 400) will *not* cause this method to raise — the VM lands in - the entity store, the phase-list ends up empty, and the - rejection only manifests when the first inference request - against that VM gets back "no factory registered for VM ..." - from the plugin. Tests that want to assert a rejection - should fire a request via :meth:`chat_completions` and check - for the error there, not assert that ``add_virtual_model`` - itself raises. + Plugin errors during upsert are swallowed. Both + ``validate_middleware_config`` and ``on_virtual_model_upserted`` + failures get logged-and-continued by the registry, so a + plugin that rejects a VM at upsert time won't make this + method raise — the VM lands in the store with an empty + phase-list and the rejection surfaces as "no factory + registered" on the first request. Assert rejections via + :meth:`chat_completions`, not via this method's return. """ vm = self._create_virtual_model( workspace=workspace, @@ -542,12 +608,12 @@ async def aadd_virtual_model( return vm def refresh_caches(self) -> None: - """Refresh model cache **and** VM cache (sync). + """Refresh model + VM cache (sync). Model cache first because VM + resolution depends on the served-model topology. - Model cache first — VM-cache resolution depends on the - served-model topology. The model-cache refresh resolves provider - secrets via the SDK; call this rather than relying on - :meth:`add_provider`'s fast path when ``api_key_secret_name`` is set. + The model-cache refresh resolves provider secrets via the SDK; + call this instead of relying on :meth:`add_provider`'s fast path + when ``api_key_secret_name`` is set. """ asyncio.run(self._refresh_all_caches()) @@ -616,26 +682,20 @@ async def _refresh_model_cache(self) -> None: # ------------------------------------------------------------------ def mock_chat_completions(self, model: str, responses: Sequence[MockResponse]) -> None: - """Queue *responses* for chat-completion calls whose ``body["model"] == model``. + """Queue *responses* for chat-completion calls with ``body["model"] == model``. - ``model`` is the value that arrives at the upstream. For a - provider with ``served_models={"main": "main"}`` and an entity - ``"default/main"``, the upstream receives ``"model": "main"``, - so register under ``"main"``. + *model* is the value that arrives at the upstream. For + ``served_models={"main": "main"}`` on a ``default/main`` entity, + the upstream sees ``"model": "main"`` — register under ``"main"``. - Plugins issuing their own outbound calls (e.g. Guardrails' rail - calls) typically send the workspace-qualified entity id — - ``"default/main"``. Register a separate queue under the entity - id for those. + Plugin-issued outbound calls (Guardrails rails, etc.) typically + send the workspace-qualified entity id like ``"default/main"``; + register a separate queue for those. - Repeated calls for the same model append; the queue consumes in - order, reusing the last response if drained. - - Response bodies built with :func:`chat_completion` or - :func:`chat_completion_chunk` that left ``model`` at the default - are automatically stamped with the dispatch key so the response - body's ``"model"`` field matches the routing key without the - caller having to repeat it. + Repeated calls append; the queue is consumed in order and the + last response is reused once drained. Response bodies built with + :func:`chat_completion` / :func:`chat_completion_chunk` that + leave ``model`` unset are auto-stamped with the dispatch key. Raises: ValueError: If *responses* is empty. @@ -692,37 +752,23 @@ def stream_chat_completions( body: dict[str, Any], extra_headers: Mapping[str, str] | None = None, ) -> list[dict[str, Any]] | dict[str, Any]: - """Call IGW's chat completions endpoint with streaming and return the parsed body. - - Forces ``stream=True`` on *body* and uses :class:`TestClient` directly - because the SDK's ``post(...)`` buffers the full body before returning, - which would defeat the purpose of streaming. - - Returns one of two shapes, depending on the response's ``Content-Type``: - - - ``text/event-stream`` → ``list[dict]`` of parsed SSE chunks in - order (the terminating ``data: [DONE]`` is dropped). This is the - normal case: upstream returned a stream and IGW relayed it. - - ``application/json`` → the raw JSON dict. This happens when a - plugin short-circuits the proxy with an :class:`ImmediateResponse` - (e.g. an input rail blocks before any tokens stream); the proxy - step never runs, so there's nothing to encode as SSE. Callers - that want to demand SSE can ``isinstance(result, list)`` at the - call site. - - Use this to integration-test ``process_response`` with an - :class:`AsyncIterator[dict]` payload, or to assert the JSON body - a plugin emits when it blocks a streaming request. Pair with a - :class:`~nmp.testing.mock_chat_completions.ChatCompletionStream` - mock response for the upstream model on the SSE branch. - - The ``/apis/inference-gateway`` prefix mirrors the production mount - path (see :func:`nmp.platform_runner.server.create_app`), which is - what :class:`create_test_client` uses too — without the prefix the - TestClient returns 404. + """Call the chat-completions endpoint with ``stream=True``. + + Uses :class:`TestClient` directly because the SDK's ``post`` + buffers the full body before returning, defeating streaming. + + Return type depends on ``Content-Type``: + + - ``text/event-stream`` → list of parsed SSE chunks in order + (``data: [DONE]`` dropped). The normal case: upstream streamed + and IGW relayed. + - ``application/json`` → the raw JSON dict. Happens when a plugin + short-circuits the proxy with :class:`ImmediateResponse` (e.g. + an input rail blocks before any token streams). Demand SSE + with ``isinstance(result, list)`` if your assertion requires it. Raises: - httpx.HTTPStatusError: If IGW returns a non-2xx status. + httpx.HTTPStatusError: On non-2xx from IGW. """ streaming_body = {**body, "stream": True} path = f"/apis/inference-gateway/v2/workspaces/{workspace}/openai/-/v1/chat/completions" @@ -762,11 +808,11 @@ def assert_request_messages_contain( *, index: int = 0, ) -> None: - """Assert the *index*-th recorded request to *model* contains *substring*. + """Assert the *index*-th request to *model* has *substring* in any message. - Searches across the ``content`` field of every message in the - request body. Raises ``AssertionError`` if no request at *index* - exists or the substring is missing. + Searches across each message's ``content`` field. Raises + ``AssertionError`` if no request at *index* or the substring is + missing. """ recorded_for_model = self.requests_for(model) if index < 0 or index >= len(recorded_for_model): @@ -798,16 +844,15 @@ def assert_request_body_for( *, index: int = 0, ) -> None: - """Assert *predicate(body)* is true for the *index*-th recorded request to *model*. + """Assert *predicate(body)* is true for the *index*-th request to *model*. - Generalises :meth:`assert_request_messages_contain` to any property of - the request body — tool calls, response_format, embedding inputs, - custom plugin-injected fields, etc. The predicate receives the parsed - JSON body verbatim. + Generalises :meth:`assert_request_messages_contain` to any body + property (tool calls, response_format, embedding inputs, + plugin-injected fields). The predicate gets the parsed JSON + body verbatim. Raises: - AssertionError: If no request at *index* exists for *model*, or - if the predicate returns falsy. + AssertionError: If no request at *index*, or predicate returns falsy. """ recorded_for_model = self.requests_for(model) if index < 0 or index >= len(recorded_for_model): @@ -827,22 +872,14 @@ def assert_request_path_for( ) -> None: """Assert the *index*-th recorded request to *model* arrived on *path*. - Path comparison is exact — a leading slash counts. The mock NIM - records the path verbatim from the inbound request, so callers - comparing against IGW-rewritten paths should include the leading - ``/`` (e.g. ``"/v1/messages"`` not ``"v1/messages"``). - - Useful for plugins that rewrite ``InferenceRequest.path`` - mid-pipeline — most prominently switchyard's ``translate`` - factory, which routes OpenAI Chat requests to the Anthropic - ``v1/messages`` endpoint by stamping - :data:`CTX_PATH_UPDATE` into the proxy context. Without a path - assertion the test only proves the in-memory rewrite happened, - not that it actually reached the upstream socket. + Exact match — include the leading slash. Useful for plugins that + rewrite ``InferenceRequest.path`` mid-pipeline (e.g. switchyard + rerouting OpenAI Chat to Anthropic ``v1/messages``); without + this you only prove the in-memory rewrite happened, not that it + reached the wire. Raises: - AssertionError: If no request at *index* exists for *model*, - or the recorded path doesn't match *path*. + AssertionError: If no request at *index*, or path mismatch. """ recorded_for_model = self.requests_for(model) if index < 0 or index >= len(recorded_for_model): @@ -861,17 +898,16 @@ def assert_request_headers_contain( *, index: int = 0, ) -> None: - """Assert the *index*-th recorded request to *model* carries header *header*. + """Assert the *index*-th request to *model* carries *header*. - Header lookup is case-insensitive (matches HTTP semantics). When - *value* is ``None``, only the header's presence is asserted; when - *value* is given, an exact match is required. Use + Case-insensitive (HTTP semantics). With *value* unset only + presence is asserted; with *value* set, an exact match. Use :meth:`requests_for` directly for substring or duplicate-header assertions. Raises: - AssertionError: If no request at *index* exists for *model*, the - header is absent, or *value* doesn't match. + AssertionError: If no request at *index*, the header is + absent, or *value* doesn't match. """ recorded_for_model = self.requests_for(model) if index < 0 or index >= len(recorded_for_model): @@ -895,31 +931,24 @@ def assert_request_headers_contain( # ------------------------------------------------------------------ async def aflush_post_response(self) -> None: - """Await every fire-and-forget post-response task IGW has scheduled so far. - - IGW schedules :func:`execute_post_response_middleware` via - :func:`asyncio.create_task` after the response has been sent to the - caller (see ``proxy.py``). The fixture initialises - ``app.state.pending_post_response_tasks = []`` and ``proxy.py`` - appends each scheduled task to that list, giving tests a - deterministic way to await them before asserting. - - **Loop constraint:** post-response tasks are bound to whichever - event loop scheduled them — typically the loop driving the inbound - request. ``aflush_post_response`` must run on the same loop. - That means tests should drive the request via :meth:`achat_completions` - (so the request and the post-response tasks share the loop) and - await this method directly. Calling it after a sync - :meth:`chat_completions` is unsupported because the SDK runs the - request on a transient loop that's already torn down by the time - the call returns; the post-response task is unreachable. + """Await every fire-and-forget post-response task IGW has scheduled. + + ``proxy.py`` appends each :func:`execute_post_response_middleware` + task to ``app.state.pending_post_response_tasks`` (set up by the + fixture). This drains and awaits them. + + **Loop constraint**: post-response tasks are bound to the loop + that scheduled them — the request loop. Drive your request from + ``async def`` via :meth:`achat_completions` so the request and + flush share a loop. Calling this after a sync + :meth:`chat_completions` doesn't work: the SDK's transient loop + is already torn down. Exceptions raised by post-response middleware are **not** raised - from here — they're swallowed inside + from here — they're swallowed in :func:`execute_post_response_middleware` (matching production's - fire-and-forget contract), but task results are awaited via - ``asyncio.gather(..., return_exceptions=True)`` so a single failure - doesn't stop the flush. + fire-and-forget contract); ``asyncio.gather(return_exceptions=True)`` + ensures one failure doesn't stop the rest of the flush. """ pending = self._pending_post_response_tasks() if pending is None: @@ -935,9 +964,7 @@ async def aflush_post_response(self) -> None: await asyncio.gather(*in_flight, return_exceptions=True) def _pending_post_response_tasks(self) -> list[asyncio.Task[None]] | None: - # ``TestClient.app`` is typed as a bare ``ASGIApp`` callable; cast so - # ``state`` (a :class:`FastAPI` attribute) is reachable to the type - # checker. The fixture is responsible for initialising the list. + # TestClient.app is typed as bare ASGIApp; cast so .state is reachable. from fastapi import FastAPI app = cast(FastAPI, self.test_client.app) @@ -945,11 +972,7 @@ def _pending_post_response_tasks(self) -> list[asyncio.Task[None]] | None: def _coerce_dict(value: Any) -> dict[str, Any]: - """Cast an SDK response (dict or Pydantic model) to ``dict``. - - Anything else is a programming error — a string error body shouldn't - pretend to be a dict. - """ + """Cast an SDK response (dict or Pydantic model) to ``dict``.""" if isinstance(value, dict): return value if hasattr(value, "model_dump"): @@ -962,11 +985,10 @@ def _coerce_dict(value: Any) -> dict[str, Any]: def _parse_sse_text(text: str) -> list[dict[str, Any]]: - """Parse a buffered SSE response string into a list of chunk dicts. + """Parse a buffered SSE response into chunk dicts. - Skips ``data: [DONE]`` and silently drops malformed JSON lines (matching - IGW's own ``_parse_sse_stream`` permissiveness — real upstreams - occasionally emit keep-alives or comments that aren't JSON). + Skips ``data: [DONE]`` and silently drops malformed JSON lines, + matching IGW's own ``_parse_sse_stream`` permissiveness. """ import json @@ -988,13 +1010,12 @@ def _parse_sse_text(text: str) -> list[dict[str, Any]]: def _instantiate_discovered_plugin(name: str) -> NemoInferenceMiddleware: - """Locate *name* in the ``nemo.inference_middleware`` entry-point group and instantiate. + """Look up *name* in the entry-point group and instantiate. - Discovery is cached at the :func:`~nemo_platform_plugin.discovery.discover` layer - for the process lifetime; tests adding/removing entry points dynamically - must call ``discover.cache_clear()`` themselves. Raises a verbose - :class:`ValueError` on miss to nudge callers toward the most common fix - (install the plugin's package, or fall back to :meth:`use_plugin`). + Discovery is cached for the process lifetime; tests adding/removing + entry points dynamically must clear the cache themselves. The + error on miss points at the two most common fixes (install the + package, or fall back to :meth:`use_plugin`). """ discovered = discover_inference_middleware() cls = discovered.get(name) @@ -1015,63 +1036,48 @@ def _instantiate_discovered_plugin(name: str) -> NemoInferenceMiddleware: class IGWLoopbackHarness(IGWPluginHarness): """:class:`IGWPluginHarness` plus IGW served on a real ``127.0.0.1`` port. - Most plugin tests should prefer :class:`IGWPluginHarness` and pin - ``parameters.base_url`` directly to :attr:`nim_base_url`. Reach for - this harness only when the test specifically needs the in-process - app reachable over a real socket — e.g. when the plugin calls + Prefer :class:`IGWPluginHarness` for most tests. Reach for this + harness only when the in-process app needs to be reachable over a + real socket — e.g. when the plugin calls :meth:`~nemo_platform_plugin.inference_middleware.InferenceMiddlewareCacheAccessor.get_openai_compatible_inference_url_and_model` - and the resulting URL must be reachable, or when plugin outbound - HTTP needs to traverse IGW's full request pipeline (VirtualModel - resolution + middleware) instead of terminating directly at the - upstream mock. + and the returned URL must actually work, or when plugin outbound + HTTP needs to go through IGW's full request pipeline instead of + landing at the upstream mock. - Costs: a uvicorn thread, two extra context-manager levels, and an - HTTP hop on every plugin-side outbound request. Opt-in for that reason. + Costs a uvicorn thread and an HTTP hop on every plugin-side + outbound request — hence opt-in. .. warning:: - **Two-loop limitation.** This harness drives the same FastAPI app - from *two* event loops simultaneously: the :class:`TestClient`'s - loop (used by the SDK's ASGI transport) and the uvicorn thread's - loop (used for plugin-originated outbound HTTP that hits the - loopback URL). Loop-bound resources are tricky here: + **Two-loop limitation.** This harness drives the FastAPI app + from two event loops: the TestClient's (for SDK requests via + ASGI transport) and uvicorn's (for plugin-originated HTTP + hitting the loopback URL). Implications: * The fixture overrides :func:`global_http_client` with a per-request :class:`aiohttp.ClientSession` so the proxy step's - HTTP client is always created on the loop handling the inbound - request. Production uses a process-singleton client tied to - service lifespan and is unaffected. - * If a plugin's :meth:`on_startup` builds a long-lived - loop-bound resource (``aiohttp.ClientSession``, ``asyncio.Lock``, - long-running ``Task``) and uses it during - :meth:`process_request`, the resource will be bound to the - startup loop and likely fail with "attached to a different loop" - when the request runs on the other loop. Such plugins should - wire their long-lived resources lazily (per-request, or behind a - loop-aware factory) — or be tested via the default - :class:`IGWPluginHarness` where only one loop is in play. - * Other production shared resources that aren't per-request - overridable (connection pools, async caches, custom - ``asyncio.Queue``) carry the same risk and may need similar - dependency overrides. - - The fixture's three patches (``per_request_http_client``, - ``override_platform_base_url``, ``serve_app_in_thread``) document - their individual reasons; the harness-level summary lives here so - callers see it at point of use. + client is created on the loop handling the request. + * A plugin that builds a long-lived loop-bound resource + (``aiohttp.ClientSession``, ``asyncio.Lock``, long-running + ``Task``) in ``on_startup`` and uses it from + ``process_request`` will fail with "attached to a different + loop" — the request loop is different. Wire those lazily, or + test via the plain :class:`IGWPluginHarness` where only one + loop is in play. + * Other shared production resources (connection pools, async + caches, ``asyncio.Queue``) carry the same risk and may need + per-request dependency overrides too. """ igw_loopback_base_url: str - """``http://:`` — bare loopback root, no path. For - workspace-scoped openai-compatible URLs use - :meth:`igw_openai_loopback_url`.""" + """``http://:`` — bare loopback root, no path. Use + :meth:`igw_openai_loopback_url` for the workspace-scoped variant.""" def igw_openai_loopback_url(self, workspace: str) -> str: - """Workspace-scoped OpenAI-compatible loopback URL. + """Workspace-scoped OpenAI-compatible loopback URL (includes ``/v1``). - Pass as ``parameters.base_url`` when plugin outbound HTTP should - traverse IGW's openai-compatible proxy for *workspace*. Includes - the ``/v1`` suffix expected by OpenAI clients. + Pass as ``parameters.base_url`` to route plugin outbound HTTP + through IGW's openai-compatible proxy for *workspace*. """ return f"{self.igw_loopback_base_url}/apis/inference-gateway/v2/workspaces/{workspace}/openai/-/v1"