diff --git a/docs/agents/deploy-agents.mdx b/docs/agents/deploy-agents.mdx index 0a62534b34..3c74746d27 100644 --- a/docs/agents/deploy-agents.mdx +++ b/docs/agents/deploy-agents.mdx @@ -335,28 +335,22 @@ for NVIDIA Build, OpenAI, and Anthropic examples. -For container modes, the gateway URL injected into the agent must be reachable -**from inside the container**, not just from the platform host. A loopback -default like `http://localhost:8080` resolves to the container itself and the -agent's model calls will fail. Set the platform's base URL to a -container-reachable address — for example the Docker bridge address -(`http://172.17.0.1:8080`) under Docker, or the in-cluster gateway address under -Kubernetes. +A deployed agent needs to reach the platform from **inside its container**. The +deployment handles this automatically for both Docker and Kubernetes, so you +normally don't need to configure anything. If an agent can't reach the platform, +set `agents.deployments.gateway_url_override` to a URL that is reachable from +inside the container. #### Docker mode on Linux -Docker Desktop (macOS/Windows) handles this automatically. On **Linux**, where -`host.docker.internal` does not resolve inside containers, point the platform at -the Docker bridge address (`172.17.0.1` by default) instead, or `nemo agents -invoke` fails with `openai.APIConnectionError`. - -Set both in `config.yaml`, then start the platform bound to all interfaces: +On **Linux**, `host.docker.internal` doesn't resolve inside containers, so agent +invokes can fail with `openai.APIConnectionError`. Point the deployment at the +Docker bridge address (`172.17.0.1` by default) in `config.yaml`, then start the +platform bound to all interfaces: ```yaml -platform: - base_url: http://172.17.0.1:8080 agents: deployments: gateway_url_override: http://172.17.0.1:8080 diff --git a/k8s/helm/templates/api/api-deployment.yaml b/k8s/helm/templates/api/api-deployment.yaml index b3bff3c8b3..5a95170bad 100644 --- a/k8s/helm/templates/api/api-deployment.yaml +++ b/k8s/helm/templates/api/api-deployment.yaml @@ -75,6 +75,10 @@ spec: {{- else }} value: {{ include "nemo-platform.internalBaseUrl" . | quote }} {{- end }} + # In-cluster API Service DNS. Deployed agents use this as their inference + # endpoint, since NMP_BASE_URL may not be reachable from an agent pod. + - name: NMP_INTERNAL_BASE_URL + value: {{ include "nemo-platform.internalBaseUrl" . | quote }} - name: NMP_AUTOMODEL_DEFAULT_TRAINING_EXECUTION_PROFILE value: {{ dig "automodel" "default_training_execution_profile" "default" $platformConfig | quote }} - name: NMP_UNSLOTH_DEFAULT_TRAINING_EXECUTION_PROFILE diff --git a/k8s/helm/templates/core/controller-deployment.yaml b/k8s/helm/templates/core/controller-deployment.yaml index 0b9204afd6..e03b3351ba 100644 --- a/k8s/helm/templates/core/controller-deployment.yaml +++ b/k8s/helm/templates/core/controller-deployment.yaml @@ -59,6 +59,10 @@ spec: value: /etc/nmp/config.yaml - name: NMP_BASE_URL value: {{ include "nemo-platform.internalBaseUrl" . | quote }} + # In-cluster API Service DNS. Deployed agents use this as their inference + # endpoint, since NMP_BASE_URL may not be reachable from an agent pod. + - name: NMP_INTERNAL_BASE_URL + value: {{ include "nemo-platform.internalBaseUrl" . | quote }} {{- if include "nemo-platform.embeddedPdpEnabled" . }} - name: NMP_AUTH_POLICY_DECISION_POINT_BASE_URL value: {{ include "nemo-platform.internalBaseUrl" . | quote }} diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/config.py b/plugins/nemo-agents/src/nemo_agents_plugin/config.py index 325254430f..d18ad6e4f9 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/config.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/config.py @@ -77,9 +77,17 @@ class DeploymentsRunnerConfig(BaseModel): gateway_url_override: str | None = Field( default=None, description=( - "Optional container-reachable platform base URL. When unset, docker mode rewrites " - "loopback hosts to host.docker.internal; k8s mode leaves the host base URL as-is " - "(in-cluster IGW DNS is AIRCORE-863)." + "Platform base URL baked into deployed agents as their inference endpoint, used verbatim " + "for both docker and k8s. When unset, the deploy path derives a container-reachable URL: " + "docker rewrites loopback hosts to host.docker.internal; k8s uses k8s_internal_base_url." + ), + ) + k8s_internal_base_url: str | None = Field( + default=None, + description=( + "In-cluster platform base URL (the API Service DNS, e.g. http://-nmp-api:8080) " + "used as the inference endpoint for k8s-mode agents. Set automatically by the Helm chart. " + "Read from NEMO_INTERNAL_BASE_URL, then NMP_INTERNAL_BASE_URL, when unset." ), ) plugin_wheels_init_image: str | None = Field( diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py b/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py index bcd3697ee1..6a685fe28c 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py @@ -14,9 +14,11 @@ from __future__ import annotations import asyncio +import copy import logging import time from typing import Any +from urllib.parse import urlsplit import yaml from nemo_agents_plugin.config import AgentsConfig, DeploymentsRunnerConfig @@ -27,7 +29,7 @@ Endpoint, ) from nemo_agents_plugin.runner.backend import DeploymentInfo, ExternalLog, LogLocation, RunnerBackend -from nemo_agents_plugin.utils import get_base_url +from nemo_agents_plugin.utils import get_base_url, get_internal_base_url from nemo_deployments_plugin.entities import ( ConfigFile, Container, @@ -79,21 +81,77 @@ def map_status(backend_status: str) -> DeploymentStatus: return _STATUS_MAP.get(backend_status, "starting") -def container_gateway_url(base_url: str, *, mode: DeploymentMode, override: str | None = None) -> str: - """Return a platform base URL reachable from inside the agent container. +class UnreachableGatewayURLError(ValueError): + """Raised when no inference base URL reachable from an agent container can be resolved.""" - Docker may rewrite loopback hosts to ``host.docker.internal``. K8s leaves the - URL as-is (in-cluster IGW Service DNS is AIRCORE-863). *override* wins verbatim. + +def resolve_agent_gateway_url( + base_url: str, + *, + mode: DeploymentMode, + override: str | None = None, + internal_base_url: str | None = None, +) -> str: + """Return the platform base URL an agent should call, reachable from its container. + + An explicit *override* wins for any mode. Otherwise ``k8s`` uses + *internal_base_url* (the in-cluster API Service DNS) and ``docker`` rewrites a + loopback *base_url* to ``host.docker.internal``, passing other hosts through. + + Only ``docker`` and ``k8s`` are supported; ``subprocess`` deployments are + served by a different backend. + + Raises: + UnreachableGatewayURLError: k8s mode with no *internal_base_url* or *override*. + ValueError: *mode* is not a container deployment mode. """ + if mode not in CONTAINER_DEPLOYMENT_MODES: + raise ValueError( + f"resolve_agent_gateway_url only supports container deployment modes " + f"{sorted(CONTAINER_DEPLOYMENT_MODES)}, got {mode!r}." + ) + if override: return override.rstrip("/") - url = base_url.rstrip("/") - if mode == "docker": - for host in LOOPBACK_ADDRESSES: - marker = f"//{host}" - if marker in url: - return url.replace(marker, "//host.docker.internal", 1) - return url + + if mode == "k8s": + if internal_base_url: + return internal_base_url.rstrip("/") + raise UnreachableGatewayURLError( + f"No container-reachable inference base URL for k8s deployment: platform base URL " + f"{base_url!r} is not usable from an agent pod and no internal API Service URL is set. " + "Set NEMO_INTERNAL_BASE_URL / NMP_INTERNAL_BASE_URL (or deployments.k8s_internal_base_url), " + "or deployments.gateway_url_override." + ) + + parts = urlsplit(base_url.rstrip("/")) + if (parts.hostname or "").lower() in LOOPBACK_ADDRESSES: + netloc = "host.docker.internal" + if parts.port is not None: + netloc = f"{netloc}:{parts.port}" + return parts._replace(netloc=netloc).geturl() + return parts.geturl() + + +def rewrite_config_base_urls(nat_config: dict[str, Any], gateway_url: str) -> dict[str, Any]: + """Return a copy of *nat_config* with each Inference Gateway LLM base_url rebased onto *gateway_url*. + + Rewrites the scheme, host, and port of ``base_url`` on every ``openai``/``nim`` + LLM that points at the Inference Gateway, preserving the path. LLMs with an + explicit third-party ``base_url`` are left unchanged. + """ + reachable = urlsplit(gateway_url.rstrip("/")) + reachable_origin = f"{reachable.scheme}://{reachable.netloc}" + config = copy.deepcopy(nat_config) + for llm_cfg in config.get("llms", {}).values(): + if not isinstance(llm_cfg, dict) or llm_cfg.get("_type") not in ("openai", "nim"): + continue + current = llm_cfg.get("base_url") + if not isinstance(current, str) or "/apis/inference-gateway/" not in current: + continue + parts = urlsplit(current) + llm_cfg["base_url"] = f"{reachable_origin}{parts.path}" + return config def executor_for_mode(config: DeploymentsRunnerConfig, mode: DeploymentMode) -> str | None: @@ -139,7 +197,6 @@ def build_deployment_config( nat_config: dict[str, Any], config_mount_path: str, mode: DeploymentMode, - gateway_base_url: str, plugin_wheels_init_image: str | None = None, labels: dict[str, str] | None = None, ) -> DeploymentConfig: @@ -150,10 +207,13 @@ def build_deployment_config( ``NAT_CONFIG_YAML`` and a shell preamble that writes the file before ``nat`` starts. The main container binds ``0.0.0.0`` and exposes a readiness probe on ``/health``. + + The inference base URL the agent calls is read from ``nat_config``'s + ``llms.*.base_url``; the caller is responsible for setting it to a + container-reachable value. """ nat_yaml = yaml.safe_dump(nat_config, sort_keys=False) env = [ - EnvVar(name="NMP_GATEWAY_BASE_URL", value=gateway_base_url), EnvVar(name="NMP_WORKSPACE", value=workspace), EnvVar(name="NMP_AGENT_NAME", value=name), EnvVar(name=_NAT_CONFIG_ENV, value=config_mount_path), @@ -292,11 +352,21 @@ async def create_deployment( ) entities = self._entity_client() - gateway = container_gateway_url( - get_base_url(), - mode=deployment_mode, - override=self._config.gateway_url_override, - ) + # The base_url injected into the agent config at agent-create time is the + # platform's own base URL, which is not necessarily reachable from inside + # the agent container. Rebase it onto a container-reachable address. + try: + internal_base_url = self._config.k8s_internal_base_url or get_internal_base_url() + gateway = resolve_agent_gateway_url( + get_base_url(), + mode=deployment_mode, + override=self._config.gateway_url_override, + internal_base_url=internal_base_url, + ) + except UnreachableGatewayURLError as exc: + logger.error("Refusing to deploy agent %r: %s", name, exc) + return DeploymentInfo(name=name, status="failed", error=str(exc)) + config = rewrite_config_base_urls(config, gateway) deployment_config = build_deployment_config( name=name, workspace=workspace, @@ -305,7 +375,6 @@ async def create_deployment( nat_config=config, config_mount_path=self._config.config_mount_path, mode=deployment_mode, - gateway_base_url=gateway, plugin_wheels_init_image=self._config.plugin_wheels_init_image, labels={ "nemo.agents/deployment": name, diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/utils.py b/plugins/nemo-agents/src/nemo_agents_plugin/utils.py index 4e7b3bba7c..7df1b7daf0 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/utils.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/utils.py @@ -171,6 +171,17 @@ def get_base_url() -> str: ) +def get_internal_base_url() -> str | None: + """Return the in-cluster platform base URL reachable from inside agent pods, or None. + + This is the API Service DNS used to reach the platform from a deployed agent + when :func:`get_base_url` is not routable from inside the container. Read from + ``NEMO_INTERNAL_BASE_URL``, then ``NMP_INTERNAL_BASE_URL``. + """ + internal = os.environ.get("NEMO_INTERNAL_BASE_URL") or os.environ.get("NMP_INTERNAL_BASE_URL") + return internal.rstrip("/") if internal else None + + def get_default_model() -> str | None: """Return the default model for the platform from the SDK context.""" from nemo_platform.config import get_context diff --git a/plugins/nemo-agents/tests/unit/test_runner_deployments.py b/plugins/nemo-agents/tests/unit/test_runner_deployments.py index c57b78d06c..bf19993ce4 100644 --- a/plugins/nemo-agents/tests/unit/test_runner_deployments.py +++ b/plugins/nemo-agents/tests/unit/test_runner_deployments.py @@ -14,10 +14,12 @@ from nemo_agents_plugin.entities import Endpoint from nemo_agents_plugin.runner.deployments_backend import ( DeploymentsRunnerBackend, + UnreachableGatewayURLError, build_deployment_config, - container_gateway_url, executor_for_mode, map_status, + resolve_agent_gateway_url, + rewrite_config_base_urls, ) from nemo_deployments_plugin.entities import Deployment, DeploymentConfig from nemo_deployments_plugin.types import Endpoint as PluginEndpoint @@ -41,18 +43,72 @@ def test_map_status(backend: str, expected: str) -> None: assert map_status(backend) == expected -def test_container_gateway_url_rewrites_loopback_for_docker() -> None: - assert container_gateway_url("http://127.0.0.1:8080", mode="docker") == "http://host.docker.internal:8080" +@pytest.mark.parametrize("host", ["127.0.0.1", "localhost", "0.0.0.0", "[::1]"]) +def test_resolve_docker_rewrites_loopback_to_host(host: str) -> None: + assert resolve_agent_gateway_url(f"http://{host}:8080", mode="docker") == "http://host.docker.internal:8080" -def test_container_gateway_url_leaves_k8s_unchanged() -> None: - assert container_gateway_url("http://127.0.0.1:8080", mode="k8s") == "http://127.0.0.1:8080" +def test_resolve_docker_passes_through_non_loopback_host() -> None: + assert resolve_agent_gateway_url("http://my-platform:8080", mode="docker") == "http://my-platform:8080" -def test_container_gateway_url_override_wins() -> None: +def test_resolve_k8s_uses_internal_base_url_regardless_of_base_url() -> None: + # k8s always returns internal_base_url, never the platform base_url. + for base_url in ( + "http://localhost:8080", + "http://nemo-platform-envoy.aire-dev.svc.cluster.local:8080", + "http://some-other-host:8080", + ): + assert ( + resolve_agent_gateway_url(base_url, mode="k8s", internal_base_url="http://nmp-api:8080/") + == "http://nmp-api:8080" + ) + + +def test_resolve_k8s_without_internal_base_url_raises() -> None: + with pytest.raises(UnreachableGatewayURLError, match="internal API Service"): + resolve_agent_gateway_url("http://localhost:8080", mode="k8s") + + +def test_resolve_rejects_subprocess_mode() -> None: + with pytest.raises(ValueError, match="container deployment modes"): + resolve_agent_gateway_url("http://localhost:8080", mode="subprocess") + + +def test_resolve_override_wins_verbatim_for_every_mode() -> None: + assert ( + resolve_agent_gateway_url("http://127.0.0.1:8080", mode="docker", override="http://igw:8080/") + == "http://igw:8080" + ) assert ( - container_gateway_url("http://127.0.0.1:8080", mode="docker", override="http://igw:8080/") == "http://igw:8080" + resolve_agent_gateway_url( + "http://localhost:8080", mode="k8s", override="http://igw:8080/", internal_base_url="http://ignored:8080" + ) + == "http://igw:8080" + ) + + +def test_rewrite_config_base_urls_rebases_igw_host() -> None: + config = { + "llms": { + "llm": { + "_type": "openai", + "base_url": "http://localhost:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1", + } + } + } + result = rewrite_config_base_urls(config, "http://nmp-api:8080") + assert result["llms"]["llm"]["base_url"] == ( + "http://nmp-api:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1" ) + # Original not mutated. + assert "localhost" in config["llms"]["llm"]["base_url"] + + +def test_rewrite_config_base_urls_leaves_third_party_base_url() -> None: + config = {"llms": {"llm": {"_type": "openai", "base_url": "https://api.openai.com/v1"}}} + result = rewrite_config_base_urls(config, "http://nmp-api:8080") + assert result["llms"]["llm"]["base_url"] == "https://api.openai.com/v1" def test_executor_for_mode_prefers_mode_specific() -> None: @@ -81,7 +137,6 @@ def test_build_deployment_config_always_single_container() -> None: nat_config={"llms": {"nim": {"_type": "nim"}}}, config_mount_path="/workspace/config.yaml", mode="docker", - gateway_base_url="http://host.docker.internal:8080", ) assert cfg.restart_policy == "Always" assert len(cfg.containers) == 1 @@ -107,7 +162,6 @@ def test_build_deployment_config_k8s_uses_nat_entrypoint() -> None: nat_config={}, config_mount_path="/workspace/config.yaml", mode="k8s", - gateway_base_url="http://nmp:8080", ) assert cfg.containers[0].command == ["nat", "start", "fastapi"] assert "--host" in cfg.containers[0].args and "0.0.0.0" in cfg.containers[0].args @@ -123,7 +177,6 @@ def test_build_deployment_config_k8s_option_b_when_image_set() -> None: nat_config={}, config_mount_path="/workspace/config.yaml", mode="k8s", - gateway_base_url="http://nmp:8080", plugin_wheels_init_image="busybox:1.36", ) assert len(cfg.init_containers) == 1 @@ -140,7 +193,6 @@ def test_build_deployment_config_docker_never_emits_init_containers() -> None: nat_config={}, config_mount_path="/workspace/config.yaml", mode="docker", - gateway_base_url="http://host.docker.internal:8080", plugin_wheels_init_image="busybox:1.36", ) assert cfg.init_containers == [] @@ -179,6 +231,77 @@ async def test_create_deployment_writes_config_and_deployment() -> None: assert created_dep.desired_state == "READY" +@pytest.mark.asyncio +async def test_create_deployment_docker_rewrites_loopback_base_url() -> None: + backend = _backend(default_image="nat:latest", default_executor="local-docker") + entities = AsyncMock() + backend._entities = entities + config = { + "llms": { + "llm": { + "_type": "openai", + "base_url": "http://localhost:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1", + } + } + } + with patch("nemo_agents_plugin.runner.deployments_backend.get_base_url", return_value="http://localhost:8080"): + info = await backend.create_deployment( + workspace="default", name="hello-dep", config=config, port=0, deployment_mode="docker" + ) + assert info.status == "starting" + created_config = entities.create.await_args_list[0].args[0] + baked = yaml.safe_load(created_config.config_files[0].content) + assert baked["llms"]["llm"]["base_url"] == ( + "http://host.docker.internal:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1" + ) + + +@pytest.mark.asyncio +async def test_create_deployment_k8s_rewrites_loopback_to_internal() -> None: + backend = _backend( + default_image="nat:latest", + default_executor="k8s", + k8s_internal_base_url="http://nmp-api:8080", + ) + entities = AsyncMock() + backend._entities = entities + config = { + "llms": { + "llm": { + "_type": "openai", + "base_url": "http://localhost:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1", + } + } + } + with patch("nemo_agents_plugin.runner.deployments_backend.get_base_url", return_value="http://localhost:8080"): + info = await backend.create_deployment( + workspace="default", name="hello-dep", config=config, port=0, deployment_mode="k8s" + ) + assert info.status == "starting" + created_config = entities.create.await_args_list[0].args[0] + baked = yaml.safe_load(created_config.config_files[0].content) + assert baked["llms"]["llm"]["base_url"] == ( + "http://nmp-api:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1" + ) + + +@pytest.mark.asyncio +async def test_create_deployment_k8s_without_internal_url_fails() -> None: + backend = _backend(default_image="nat:latest", default_executor="k8s") + entities = AsyncMock() + backend._entities = entities + with ( + patch("nemo_agents_plugin.runner.deployments_backend.get_base_url", return_value="http://localhost:8080"), + patch("nemo_agents_plugin.runner.deployments_backend.get_internal_base_url", return_value=None), + ): + info = await backend.create_deployment( + workspace="default", name="hello-dep", config={}, port=0, deployment_mode="k8s" + ) + assert info.status == "failed" + assert "internal api service" in info.error.lower() + entities.create.assert_not_awaited() + + @pytest.mark.asyncio async def test_create_deployment_missing_image_fails() -> None: backend = _backend(default_image="") diff --git a/plugins/nemo-agents/tests/unit/test_utils.py b/plugins/nemo-agents/tests/unit/test_utils.py index 75f3c7a100..166577a34c 100644 --- a/plugins/nemo-agents/tests/unit/test_utils.py +++ b/plugins/nemo-agents/tests/unit/test_utils.py @@ -28,6 +28,7 @@ from nemo_agents_plugin.jobs.optimize_agent import OptimizeAgentJob, OptimizeAgentSpec from nemo_agents_plugin.refs import AgentRef from nemo_agents_plugin.utils import ( + get_internal_base_url, inject_default_model, inject_gateway_url, merge_agent_config, @@ -139,6 +140,23 @@ def test_trailing_slash_stripped_from_base_url(self) -> None: assert "//apis" not in result["llms"]["llm"]["base_url"] +class TestGetInternalBaseUrl: + def test_returns_none_when_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NEMO_INTERNAL_BASE_URL", raising=False) + monkeypatch.delenv("NMP_INTERNAL_BASE_URL", raising=False) + assert get_internal_base_url() is None + + def test_reads_nmp_internal_base_url(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NEMO_INTERNAL_BASE_URL", raising=False) + monkeypatch.setenv("NMP_INTERNAL_BASE_URL", "http://nmp-api:8080/") + assert get_internal_base_url() == "http://nmp-api:8080" + + def test_nemo_internal_base_url_takes_precedence(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NEMO_INTERNAL_BASE_URL", "http://nemo:8080") + monkeypatch.setenv("NMP_INTERNAL_BASE_URL", "http://nmp:8080") + assert get_internal_base_url() == "http://nemo:8080" + + # --------------------------------------------------------------------------- # inject_default_model # ---------------------------------------------------------------------------