diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/config.py b/plugins/nemo-agents/src/nemo_agents_plugin/config.py index 9db7303ec2..262aa2aaef 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/config.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/config.py @@ -98,13 +98,13 @@ class DeploymentsRunnerConfig(BaseModel): ), ) config_mount_path: str = Field( - default="/workspace/config.yaml", + default="/tmp/nemo/config.yaml", description=( "Path inside the container where the NAT workflow config is placed for nat-workflow-v1 " - "deployments. Fabric deployments use agent.yaml in the same directory. Must sit under " - "the image's writable WORKDIR (/workspace) so docker mode, which materializes the " - "config as the non-root container user, can write it; k8s mounts it read-only there " - "via a ConfigMap subPath." + "deployments. Fabric deployments use agent.yaml in the same directory. Must be writable " + "by every runtime user this image can run as: the plain-docker container user and the " + "openshell sandbox user (uid 999, which cannot write the image's /workspace). /tmp is " + "the writable intersection; k8s mounts it read-only there via a ConfigMap subPath." ), ) diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py b/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py index 7d4e5932fb..af49b2ac74 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py @@ -14,11 +14,8 @@ from __future__ import annotations import asyncio -import base64 import copy -import json import logging -import shlex import time from pathlib import PurePosixPath from typing import Any @@ -67,10 +64,7 @@ _PLUGIN_WHEELS_VOLUME = "plugin-wheels" _PLUGIN_WHEELS_MOUNT = "/opt/nemo/plugin-wheels" _NAT_CONFIG_ENV = "NAT_CONFIG_PATH" -_NAT_CONFIG_YAML_ENV = "NAT_CONFIG_YAML" -_AGENT_CONFIG_YAML_ENV = "AGENT_CONFIG_YAML" _AGENT_CONFIG_PATH_ENV = "AGENT_CONFIG_PATH" -_STAGED_CONFIG_FILES_ENV = "STAGED_CONFIG_FILES_B64_JSON" _FABRIC_SERVER_MODULE = "nemo_agents_plugin.fabric.server" _AUTH_PROXY_IDENTITY = "agents" @@ -256,29 +250,6 @@ def _fabric_server_cli_args(*, config_path: str, port: int) -> list[str]: ] -def _materialize_config_and_exec(*, config_path: str, yaml_env: str, argv: list[str]) -> list[str]: - """Return ``sh -c`` args that write the config from *yaml_env*, then exec *argv*. - - Docker mode needs this because the docker backend does not mount ``config_files``. - Paths and argv are shell-escaped so spaces/metacharacters cannot break the script. - """ - quoted_path = shlex.quote(config_path) - quoted_argv = " ".join(shlex.quote(arg) for arg in argv) - return [f'mkdir -p "$(dirname {quoted_path})" && printf "%s" "${yaml_env}" > {quoted_path} && exec {quoted_argv}'] - - -def _materialize_staged_config_files_and_exec(*, env_name: str, argv: list[str]) -> list[str]: - """Return ``sh -c`` args that write staged ``config_files`` from *env_name*, then exec *argv*.""" - quoted_argv = " ".join(shlex.quote(arg) for arg in argv) - inline_python = ( - "import base64,json,os,pathlib;" - f"data=json.loads(os.environ[{json.dumps(env_name)}]);" - "[(pathlib.Path(p).parent.mkdir(parents=True,exist_ok=True)," - "pathlib.Path(p).write_bytes(base64.b64decode(b))) for p,b in data.items()]" - ) - return [f"python -c {shlex.quote(inline_python)} && exec {quoted_argv}"] - - def executor_for_mode(config: DeploymentsRunnerConfig, mode: DeploymentMode) -> str | None: """Resolve the named deployments-plugin executor for *mode*.""" if mode == "docker": @@ -393,11 +364,9 @@ def build_deployment_config( env.append(EnvVar(name="PYTHONPATH", value=_PLUGIN_WHEELS_MOUNT)) if is_fabric: - config_yaml_env = _AGENT_CONFIG_YAML_ENV server_command = ["python"] server_args = _fabric_server_cli_args(config_path=config_path, port=port) else: - config_yaml_env = _NAT_CONFIG_YAML_ENV server_command = ["nat", "start", "fastapi"] server_args = [ "--config_file", @@ -408,31 +377,8 @@ def build_deployment_config( str(port), ] - if mode == "docker": - # Docker backend does not mount config_files; materialize staged files from env. - if len(resolved_config_files) == 1: - single = resolved_config_files[0] - env.append(EnvVar(name=config_yaml_env, value=single.content)) - command = ["sh", "-c"] - args = _materialize_config_and_exec( - config_path=single.path, - yaml_env=config_yaml_env, - argv=[*server_command, *server_args], - ) - else: - payload = { - config_file.path: base64.b64encode(config_file.content.encode("utf-8")).decode("ascii") - for config_file in resolved_config_files - } - env.append(EnvVar(name=_STAGED_CONFIG_FILES_ENV, value=json.dumps(payload, separators=(",", ":")))) - command = ["sh", "-c"] - args = _materialize_staged_config_files_and_exec( - env_name=_STAGED_CONFIG_FILES_ENV, - argv=[*server_command, *server_args], - ) - else: - command = server_command - args = server_args + command = server_command + args = server_args container = Container( name=_CONTAINER_NAME, diff --git a/plugins/nemo-agents/tests/unit/test_runner_deployments.py b/plugins/nemo-agents/tests/unit/test_runner_deployments.py index 48dfcfb3aa..14d8cd8eef 100644 --- a/plugins/nemo-agents/tests/unit/test_runner_deployments.py +++ b/plugins/nemo-agents/tests/unit/test_runner_deployments.py @@ -217,10 +217,7 @@ def test_executor_for_mode_prefers_mode_specific() -> None: def test_config_mount_path_default_is_under_writable_workspace() -> None: - # Docker mode materializes the config as the non-root container user, so the - # default must live under the image's writable WORKDIR (/workspace); a - # root-level path like /config is not writable and crash-loops the container. - assert DeploymentsRunnerConfig().config_mount_path.startswith("/workspace/") + assert DeploymentsRunnerConfig().config_mount_path.startswith("/tmp/nemo/") def test_build_deployment_config_always_single_container() -> None: @@ -231,21 +228,20 @@ def test_build_deployment_config_always_single_container() -> None: port=8000, agent_config={"llms": {"nim": {"_type": "nim"}}}, platform_base_url="http://host.docker.internal:8080", - config_mount_path="/workspace/config.yaml", + config_mount_path="/tmp/nemo/config.yaml", mode="docker", ) assert cfg.restart_policy == "Always" assert len(cfg.containers) == 1 container = cfg.containers[0] assert container.image == "nat-runtime:latest" - # Docker materializes config from NAT_CONFIG_YAML because config_files are not mounted. - assert container.command == ["sh", "-c"] - assert any(e.name == "NAT_CONFIG_YAML" for e in container.env) + assert container.command == ["nat", "start", "fastapi"] + assert not any(e.name == "NAT_CONFIG_YAML" for e in container.env) assert next(e.value for e in container.env if e.name == "NMP_BASE_URL") == "http://host.docker.internal:8080" assert container.readiness_probe is not None assert cfg.init_containers == [] assert len(cfg.config_files) == 1 - assert cfg.config_files[0].path == "/workspace/config.yaml" + assert cfg.config_files[0].path == "/tmp/nemo/config.yaml" loaded = yaml.safe_load(cfg.config_files[0].content) assert loaded["llms"]["nim"]["_type"] == "nim" @@ -316,22 +312,6 @@ def test_build_deployment_config_docker_never_emits_init_containers() -> None: } -def test_build_deployment_config_docker_shell_escapes_config_path() -> None: - cfg = build_deployment_config( - name="spaced-dep", - workspace="default", - image="nat-runtime:latest", - port=8000, - agent_config={"llms": {"nim": {"_type": "nim"}}}, - platform_base_url="http://host.docker.internal:8080", - config_mount_path="/workspace/my config/config.yaml", - mode="docker", - ) - script = cfg.containers[0].args[0] - assert "'/workspace/my config/config.yaml'" in script - assert 'printf "%s" "$NAT_CONFIG_YAML"' in script - - def test_build_deployment_config_fabric_docker_uses_fabric_server() -> None: cfg = build_deployment_config( name="fabric-dep", @@ -340,23 +320,27 @@ def test_build_deployment_config_fabric_docker_uses_fabric_server() -> None: port=8000, agent_config=_FABRIC_AGENT_CONFIG, platform_base_url="http://host.docker.internal:8080", - config_mount_path="/workspace/config.yaml", + config_mount_path="/tmp/nemo/config.yaml", mode="docker", ) container = cfg.containers[0] - assert container.command == ["sh", "-c"] - assert any(e.name == "AGENT_CONFIG_YAML" for e in container.env) + assert container.command == ["python"] + assert container.args[0] == "-m" + assert container.args[1] == "nemo_agents_plugin.fabric.server" + assert "--agent-config" in container.args + assert "/tmp/nemo/agent.yaml" in container.args + assert "--host" in container.args and "0.0.0.0" in container.args + assert not any(e.name == "AGENT_CONFIG_YAML" for e in container.env) assert not any(e.name == "NAT_CONFIG_YAML" for e in container.env) - assert any(e.name == "AGENT_CONFIG_PATH" and e.value == "/workspace/agent.yaml" for e in container.env) + assert any(e.name == "AGENT_CONFIG_PATH" and e.value == "/tmp/nemo/agent.yaml" for e in container.env) assert next(e.value for e in container.env if e.name == "NMP_BASE_URL") == "http://host.docker.internal:8080" assert next(e.value for e in container.env if e.name == PLATFORM_IGW_API_KEY_ENV) == ( PLATFORM_IGW_API_KEY_PLACEHOLDER ) - assert "nemo_agents_plugin.fabric.server" in container.args[0] + assert cfg.config_files[0].path == "/tmp/nemo/agent.yaml" assert container.readiness_probe is not None assert container.readiness_probe.http_get is not None assert container.readiness_probe.http_get.path == "/health" - assert cfg.config_files[0].path == "/workspace/agent.yaml" def test_build_deployment_config_fabric_k8s_uses_fabric_entrypoint() -> None: @@ -414,11 +398,11 @@ def test_build_deployment_config_fabric_direct_endpoint_has_no_placeholder() -> assert not any(e.name in {PLATFORM_IGW_API_KEY_ENV, "OPENAI_API_KEY"} for e in cfg.containers[0].env) -def test_build_deployment_config_fabric_docker_materializes_multiple_config_files() -> None: +def test_build_deployment_config_fabric_docker_mounts_multiple_config_files() -> None: staged_files = [ - ConfigFile(path="/workspace/agent.yaml", content="name: fabric-agent\n"), - ConfigFile(path="/workspace/skills/review/SKILL.md", content="# Review\n"), - ConfigFile(path="/workspace/prompts/system.md", content="You are helpful.\n"), + ConfigFile(path="/tmp/nemo/agent.yaml", content="name: fabric-agent\n"), + ConfigFile(path="/tmp/nemo/skills/review/SKILL.md", content="# Review\n"), + ConfigFile(path="/tmp/nemo/prompts/system.md", content="You are helpful.\n"), ] cfg = build_deployment_config( name="fabric-dep", @@ -427,21 +411,18 @@ def test_build_deployment_config_fabric_docker_materializes_multiple_config_file port=8000, agent_config=_FABRIC_AGENT_CONFIG, platform_base_url="http://host.docker.internal:8080", - config_mount_path="/workspace/config.yaml", + config_mount_path="/tmp/nemo/config.yaml", mode="docker", config_files=staged_files, ) container = cfg.containers[0] - assert container.command == ["sh", "-c"] - assert not any(e.name == "AGENT_CONFIG_YAML" for e in container.env) - assert any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in container.env) - assert "python -c" in container.args[0] - assert "nemo_agents_plugin.fabric.server" in container.args[0] + assert container.command == ["python"] + assert not any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in container.env) assert len(cfg.config_files) == 3 assert {item.path for item in cfg.config_files} == { - "/workspace/agent.yaml", - "/workspace/skills/review/SKILL.md", - "/workspace/prompts/system.md", + "/tmp/nemo/agent.yaml", + "/tmp/nemo/skills/review/SKILL.md", + "/tmp/nemo/prompts/system.md", } @@ -746,7 +727,8 @@ async def test_create_deployment_fabric_docker_rewrites_model_base_url() -> None "http://host.docker.internal:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1" ) assert created_config.labels["nemo.agents/runtime"] == "fabric" - assert "nemo_agents_plugin.fabric.server" in created_config.containers[0].args[0] + assert created_config.containers[0].command == ["python"] + assert "nemo_agents_plugin.fabric.server" in created_config.containers[0].args @pytest.mark.asyncio @@ -997,7 +979,7 @@ async def test_create_deployment_fabric_docker_stages_fileset_artifacts() -> Non mock_stage.assert_awaited_once() created_config = entities.create.await_args_list[0].args[0] assert len(created_config.config_files) == 2 - assert any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in created_config.containers[0].env) + assert not any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in created_config.containers[0].env) @pytest.mark.asyncio diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py index 641defb6db..e5cd0d9021 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py @@ -6,8 +6,10 @@ from __future__ import annotations import asyncio +import io import logging import os +import tarfile from typing import TYPE_CHECKING, Any from nemo_deployments_plugin.backends.base import ( @@ -57,7 +59,7 @@ managed_by_filter, ) from nemo_deployments_plugin.constants import MANAGED_BY_LABEL -from nemo_deployments_plugin.entities import Container, Deployment, DeploymentConfig +from nemo_deployments_plugin.entities import ConfigFile, Container, Deployment, DeploymentConfig from nemo_deployments_plugin.secrets import SecretResolutionError, resolve_deployment_config_secrets from nemo_deployments_plugin.types import Endpoint, RestartPolicy from nemo_platform_plugin.capabilities import docker_from_env_kwargs, probe_docker @@ -93,6 +95,30 @@ def _is_ngc_image(image: str) -> bool: return image == NGC_IMAGE_REGISTRY or image.startswith(f"{NGC_IMAGE_REGISTRY}/") +def _config_files_tar(config_files: list[ConfigFile]) -> bytes: + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + seen_dirs: set[str] = set() + for cf in config_files: + rel = cf.path.lstrip("/") + parts = rel.split("/") + for i in range(1, len(parts)): + d = "/".join(parts[:i]) + if d in seen_dirs: + continue + seen_dirs.add(d) + info = tarfile.TarInfo(name=d) + info.type = tarfile.DIRTYPE + info.mode = 0o755 + tar.addfile(info) + data = cf.content.encode("utf-8") + info = tarfile.TarInfo(name=rel) + info.size = len(data) + info.mode = 0o644 + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + class DockerDeploymentBackend(DeploymentBackend): """Manage deployments and volumes as Docker containers and volumes.""" @@ -329,7 +355,9 @@ async def create_deployment( network=f"container:{c_name}", ) try: - await asyncio.to_thread(self._client.containers.run, **sidecar_run_kwargs) + sidecar_create_kwargs = {k: v for k, v in sidecar_run_kwargs.items() if k != "detach"} + sidecar_container = await asyncio.to_thread(self._client.containers.create, **sidecar_create_kwargs) + await asyncio.to_thread(sidecar_container.start) except Exception as exc: logger.exception("Failed to start sidecar container %s", sidecar_name) # Tear the whole group down so we don't leave a half-started deployment. @@ -421,10 +449,15 @@ async def _run_server_container( gpu_ids=gpu_ids, network=network, ) + create_kwargs = {k: v for k, v in run_kwargs.items() if k != "detach"} try: - container = await asyncio.to_thread(self._client.containers.run, **run_kwargs) + container = await asyncio.to_thread(self._client.containers.create, **create_kwargs) + if config.config_files: + await self._deliver_config_files(container, config.config_files) + await asyncio.to_thread(container.start) return container, host_ports, "" except Exception as exc: + await self._remove_container_by_name(name) last_attempt = attempt == _PORT_CONFLICT_ATTEMPTS if not host_ports or last_attempt or _PORT_CONFLICT_MARKER not in str(exc): logger.exception("Failed to start container %s", name) @@ -438,9 +471,6 @@ async def _run_server_container( _PORT_CONFLICT_ATTEMPTS, sorted(rejected_ports), ) - # containers.run() creates then starts, so a failed start leaves the - # created container holding the name and blocking the retry. - await self._remove_container_by_name(name) try: reallocated = await self._allocate_host_ports(container_spec, exclude_ports=rejected_ports) except PortEnumerationError as port_exc: @@ -449,6 +479,14 @@ async def _run_server_container( return None, host_ports, "No host ports available in configured range" host_ports = reallocated + async def _deliver_config_files( + self, + container: DockerContainer, + config_files: list[ConfigFile], + ) -> None: + archive = _config_files_tar(config_files) + await asyncio.to_thread(container.put_archive, "/", archive) + def _build_run_kwargs( self, *, @@ -553,7 +591,9 @@ async def _run_init_container( run_kwargs["volumes"] = volume_bindings def _run_and_wait() -> int: - container = self._client.containers.run(**run_kwargs) + create_kwargs = {k: v for k, v in run_kwargs.items() if k != "detach"} + container = self._client.containers.create(**create_kwargs) + container.start() result = container.wait(timeout=self._executor_config.docker_timeout) exit_code = self._exit_code_from_wait_result(result) try: diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py index ea211039cc..13b58dd052 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py @@ -76,7 +76,7 @@ async def test_create_deployment_starts_container( ) -> None: mock_entities.get.return_value = sample_config() mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = MagicMock(id="abc123") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") update = await docker_backend.create_deployment( workspace="default", @@ -87,7 +87,7 @@ async def test_create_deployment_starts_container( ) assert update.status == "STARTING" - mock_docker_client.containers.run.assert_called_once() + mock_docker_client.containers.create.assert_called_once() mock_entities.get.assert_awaited() @@ -105,7 +105,7 @@ async def test_create_deployment_maps_command_to_entrypoint( """ mock_entities.get.return_value = sample_config() # command=["echo"], args=["hello"] mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = MagicMock(id="abc123") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") await docker_backend.create_deployment( workspace="default", @@ -115,9 +115,9 @@ async def test_create_deployment_maps_command_to_entrypoint( backend_config={}, ) - _, run_kwargs = mock_docker_client.containers.run.call_args - assert run_kwargs["entrypoint"] == ["echo"] - assert run_kwargs["command"] == ["hello"] + _, create_kwargs = mock_docker_client.containers.create.call_args + assert create_kwargs["entrypoint"] == ["echo"] + assert create_kwargs["command"] == ["hello"] def _port_conflict_error(port: int) -> APIError: @@ -147,12 +147,11 @@ async def test_create_deployment_reallocates_port_after_docker_conflict( """Docker's port reservations are invisible to the probe, so a publish can still lose a race.""" first_port = docker_backend._executor_config.port_range_start leftover = MagicMock() + server_container = MagicMock(id="abc123") mock_entities.get.return_value = published_port_config() mock_docker_client.containers.get.side_effect = [NotFound("missing"), leftover] - mock_docker_client.containers.run.side_effect = [ - _port_conflict_error(first_port), - MagicMock(id="abc123"), - ] + mock_docker_client.containers.create.return_value = server_container + server_container.start.side_effect = [_port_conflict_error(first_port), None] update = await docker_backend.create_deployment( workspace="default", @@ -163,8 +162,7 @@ async def test_create_deployment_reallocates_port_after_docker_conflict( ) assert update.status == "STARTING" - assert _published_host_ports(mock_docker_client.containers.run) == [first_port, first_port + 1] - # run() creates then starts, so the container that failed to start still holds the name. + assert _published_host_ports(mock_docker_client.containers.create) == [first_port, first_port + 1] leftover.remove.assert_called_once_with(force=True) @@ -176,9 +174,11 @@ async def test_create_deployment_fails_after_repeated_port_conflicts( free_host_ports: None, ) -> None: first_port = docker_backend._executor_config.port_range_start + server_container = MagicMock(id="abc123") mock_entities.get.return_value = published_port_config() mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.side_effect = [ + mock_docker_client.containers.create.return_value = server_container + server_container.start.side_effect = [ _port_conflict_error(first_port + offset) for offset in range(_PORT_CONFLICT_ATTEMPTS) ] @@ -192,7 +192,7 @@ async def test_create_deployment_fails_after_repeated_port_conflicts( assert update.status == "FAILED" assert _PORT_CONFLICT_MARKER in (update.status_message or "") - published = _published_host_ports(mock_docker_client.containers.run) + published = _published_host_ports(mock_docker_client.containers.create) assert published == [first_port + offset for offset in range(_PORT_CONFLICT_ATTEMPTS)] @@ -203,9 +203,11 @@ async def test_create_deployment_does_not_retry_unrelated_start_failure( mock_docker_client: MagicMock, free_host_ports: None, ) -> None: + server_container = MagicMock(id="abc123") mock_entities.get.return_value = published_port_config() mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.side_effect = APIError("no such image") + mock_docker_client.containers.create.return_value = server_container + server_container.start.side_effect = APIError("no such image") update = await docker_backend.create_deployment( workspace="default", @@ -216,7 +218,7 @@ async def test_create_deployment_does_not_retry_unrelated_start_failure( ) assert update.status == "FAILED" - mock_docker_client.containers.run.assert_called_once() + mock_docker_client.containers.create.assert_called_once() @pytest.mark.asyncio @@ -233,13 +235,11 @@ async def test_create_lora_group_runs_init_server_and_sidecar( mock_entities.get.return_value = lora_config() mock_docker_client.containers.get.side_effect = NotFound("missing") - # Init container is run+waited: containers.run returns a container whose - # wait() reports success. init_container = MagicMock() init_container.wait.return_value = {"StatusCode": 0} server_container = MagicMock(id="server123") sidecar_container = MagicMock(id="sidecar123") - mock_docker_client.containers.run.side_effect = [init_container, server_container, sidecar_container] + mock_docker_client.containers.create.side_effect = [init_container, server_container, sidecar_container] update = await docker_backend.create_deployment( workspace="default", @@ -250,21 +250,19 @@ async def test_create_lora_group_runs_init_server_and_sidecar( ) assert update.status == "STARTING" - calls = mock_docker_client.containers.run.call_args_list + calls = mock_docker_client.containers.create.call_args_list assert len(calls) == 3 - # 1) init container ran to completion (detached then waited + removed) + init_container.start.assert_called_once() init_container.wait.assert_called_once() init_container.remove.assert_called_once() - # 2) server publishes ports, no shared netns server_kwargs = calls[1].kwargs assert server_kwargs["name"] == container_name("default", "srv") assert server_kwargs["labels"][CONTAINER_ROLE_LABEL] == "server" assert "ports" in server_kwargs assert server_kwargs.get("network", "") == "" - # 3) sidecar shares the server netns, publishes no ports sidecar_kwargs = calls[2].kwargs assert sidecar_kwargs["name"] == companion_container_name("default", "srv", "lora-adapters") assert sidecar_kwargs["labels"][CONTAINER_ROLE_LABEL] == "lora-adapters" @@ -284,7 +282,7 @@ async def test_create_lora_group_fails_when_init_nonzero( init_container = MagicMock() init_container.wait.return_value = {"StatusCode": 1} - mock_docker_client.containers.run.return_value = init_container + mock_docker_client.containers.create.return_value = init_container update = await docker_backend.create_deployment( workspace="default", @@ -296,8 +294,7 @@ async def test_create_lora_group_fails_when_init_nonzero( assert update.status == "FAILED" assert "init" in update.status_message.lower() - # only the init container was run (server/sidecar never started) - assert mock_docker_client.containers.run.call_count == 1 + assert mock_docker_client.containers.create.call_count == 1 @pytest.mark.asyncio @@ -327,7 +324,7 @@ async def test_create_falls_back_to_local_image_when_pull_fails( mock_docker_client.containers.get.side_effect = NotFound("missing") mock_docker_client.images.pull.side_effect = APIError("404 not found") mock_docker_client.images.get.return_value = MagicMock() # present locally - mock_docker_client.containers.run.return_value = MagicMock(id="abc123") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") update = await backend.create_deployment( workspace="default", @@ -339,7 +336,7 @@ async def test_create_falls_back_to_local_image_when_pull_fails( assert update.status == "STARTING" mock_docker_client.images.get.assert_called_once() - mock_docker_client.containers.run.assert_called_once() + mock_docker_client.containers.create.assert_called_once() @pytest.mark.asyncio @@ -375,7 +372,7 @@ async def test_create_fails_when_pull_fails_and_no_local_image( assert update.status == "FAILED" assert "pull image" in update.status_message.lower() - mock_docker_client.containers.run.assert_not_called() + mock_docker_client.containers.create.assert_not_called() @pytest.mark.asyncio @@ -497,7 +494,7 @@ async def test_create_lora_group_does_not_remove_foreign_stale_init_container( sidecar_container = MagicMock(id="sidecar123") mock_entities.get.return_value = lora_config() mock_docker_client.containers.get.side_effect = [NotFound("missing"), foreign_stale] - mock_docker_client.containers.run.side_effect = [init_container, server_container, sidecar_container] + mock_docker_client.containers.create.side_effect = [init_container, server_container, sidecar_container] update = await backend.create_deployment( workspace="default", @@ -960,7 +957,7 @@ async def test_create_never_job_returns_succeeded_when_container_exits_immediate ) -> None: mock_entities.get.return_value = sample_config(restart_policy="Never") mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = _one_shot_server_container( + mock_docker_client.containers.create.return_value = _one_shot_server_container( restart_policy="Never", exit_code=0, ) @@ -975,7 +972,7 @@ async def test_create_never_job_returns_succeeded_when_container_exits_immediate assert update.status == "SUCCEEDED" assert update.exit_code == 0 - mock_docker_client.containers.run.return_value.wait.assert_called_once_with(timeout=5) + mock_docker_client.containers.create.return_value.wait.assert_called_once_with(timeout=5) @pytest.mark.asyncio @@ -998,7 +995,7 @@ async def test_create_never_job_uses_configured_oneshot_observe_timeout( mock_entities.get.return_value = sample_config(restart_policy="Never") mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = _one_shot_server_container( + mock_docker_client.containers.create.return_value = _one_shot_server_container( restart_policy="Never", exit_code=0, ) @@ -1012,7 +1009,7 @@ async def test_create_never_job_uses_configured_oneshot_observe_timeout( ) assert update.status == "SUCCEEDED" - mock_docker_client.containers.run.return_value.wait.assert_called_once_with(timeout=7) + mock_docker_client.containers.create.return_value.wait.assert_called_once_with(timeout=7) @pytest.mark.asyncio @@ -1023,7 +1020,7 @@ async def test_create_never_job_returns_failed_on_non_zero_exit( ) -> None: mock_entities.get.return_value = sample_config(restart_policy="Never") mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = _one_shot_server_container( + mock_docker_client.containers.create.return_value = _one_shot_server_container( restart_policy="Never", exit_code=42, ) @@ -1049,7 +1046,7 @@ async def test_create_on_failure_returns_succeeded_when_already_exited_zero( mock_entities.get.return_value = sample_config(restart_policy="OnFailure") mock_docker_client.containers.get.side_effect = NotFound("missing") server = _one_shot_server_container(restart_policy="OnFailure", exit_code=0) - mock_docker_client.containers.run.return_value = server + mock_docker_client.containers.create.return_value = server update = await docker_backend.create_deployment( workspace="default", @@ -1078,7 +1075,7 @@ async def test_create_on_failure_returns_starting_when_failed_under_backoff( restart_count=2, backoff_limit="6", ) - mock_docker_client.containers.run.return_value = server + mock_docker_client.containers.create.return_value = server update = await docker_backend.create_deployment( workspace="default", @@ -1102,7 +1099,7 @@ async def test_create_on_failure_returns_starting_when_still_running( mock_entities.get.return_value = sample_config(restart_policy="OnFailure") mock_docker_client.containers.get.side_effect = NotFound("missing") server = _one_shot_server_container(restart_policy="OnFailure", status="running", exit_code=0) - mock_docker_client.containers.run.return_value = server + mock_docker_client.containers.create.return_value = server update = await docker_backend.create_deployment( workspace="default", @@ -1127,7 +1124,7 @@ async def test_create_never_job_returns_starting_when_wait_times_out( mock_docker_client.containers.get.side_effect = NotFound("missing") server = _one_shot_server_container(restart_policy="Never", exit_code=0) server.wait.side_effect = ReadTimeout("timed out") - mock_docker_client.containers.run.return_value = server + mock_docker_client.containers.create.return_value = server update = await docker_backend.create_deployment( workspace="default", @@ -1152,7 +1149,7 @@ async def test_create_never_job_returns_starting_when_wait_connection_error( mock_docker_client.containers.get.side_effect = NotFound("missing") server = _one_shot_server_container(restart_policy="Never", exit_code=0) server.wait.side_effect = RequestsConnectionError("connection reset") - mock_docker_client.containers.run.return_value = server + mock_docker_client.containers.create.return_value = server update = await docker_backend.create_deployment( workspace="default", @@ -1177,7 +1174,7 @@ async def test_create_never_job_cleans_up_on_wait_error( mock_docker_client.containers.get.side_effect = NotFound("missing") server = _one_shot_server_container(restart_policy="Never", exit_code=0) server.wait.side_effect = RuntimeError("boom") - mock_docker_client.containers.run.return_value = server + mock_docker_client.containers.create.return_value = server with patch.object( docker_backend, @@ -1205,7 +1202,7 @@ async def test_create_always_still_returns_starting( ) -> None: mock_entities.get.return_value = sample_config(restart_policy="Always") mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = MagicMock(id="abc123") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") update = await docker_backend.create_deployment( workspace="default", @@ -1216,7 +1213,7 @@ async def test_create_always_still_returns_starting( ) assert update.status == "STARTING" - mock_docker_client.containers.run.return_value.wait.assert_not_called() + mock_docker_client.containers.create.return_value.wait.assert_not_called() @pytest.mark.asyncio diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py index de1269f03b..ae832ed02e 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py @@ -58,7 +58,7 @@ async def test_create_existing_matching_container_returns_read_status( ) assert update.status == "READY" - mock_docker_client.containers.run.assert_not_called() + mock_docker_client.containers.create.assert_not_called() @pytest.mark.asyncio @@ -85,7 +85,7 @@ async def test_create_exited_one_shot_container_is_removed_and_recreated( fresh.wait.return_value = {"StatusCode": 0} else: fresh.status = "running" - mock_docker_client.containers.run.return_value = fresh + mock_docker_client.containers.create.return_value = fresh update = await docker_backend.create_deployment( workspace="default", @@ -103,7 +103,7 @@ async def test_create_exited_one_shot_container_is_removed_and_recreated( assert update.status == "STARTING" fresh.wait.assert_not_called() existing.remove.assert_called_once_with(force=True) - mock_docker_client.containers.run.assert_called_once() + mock_docker_client.containers.create.assert_called_once() @pytest.mark.asyncio @@ -130,7 +130,7 @@ async def test_create_running_one_shot_container_still_returns_status( ) assert update.status == "STARTING" - mock_docker_client.containers.run.assert_not_called() + mock_docker_client.containers.create.assert_not_called() existing.remove.assert_not_called() @@ -155,4 +155,4 @@ async def test_create_exited_one_shot_returns_failed_when_removal_fails( assert update.status == "FAILED" assert "remove exited container" in update.status_message - mock_docker_client.containers.run.assert_not_called() + mock_docker_client.containers.create.assert_not_called() diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_image_pull_auth.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_image_pull_auth.py index be1f38995d..7304777ae3 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_image_pull_auth.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_image_pull_auth.py @@ -53,7 +53,7 @@ async def test_create_pulls_nvcr_image_with_ngc_auth( ) -> None: mock_entities.get.return_value = _nim_config() mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = MagicMock(id="abc123") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") with patch( "nemo_deployments_plugin.backends.docker.backend.resolve_deployment_config_secrets", @@ -80,7 +80,7 @@ async def test_create_pulls_non_ngc_image_without_auth( ) -> None: mock_entities.get.return_value = sample_config() mock_docker_client.containers.get.side_effect = NotFound("missing") - mock_docker_client.containers.run.return_value = MagicMock(id="abc123") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") update = await docker_backend_pull.create_deployment( workspace="default",