NVIDIA-NeMo · maxdubrinsky · Aug 10, 2026
@@ -98,13 +98,13 @@ class DeploymentsRunnerConfig(BaseModel):
         ),
     )
     config_mount_path: str = Field(
-        default="/workspace/config.yaml",
+        default="/tmp/nemo/config.yaml",
         description=(
             "Path inside the container where the NAT workflow config is placed for nat-workflow-v1 "
-            "deployments. Fabric deployments use agent.yaml in the same directory. Must sit under "
-            "the image's writable WORKDIR (/workspace) so docker mode, which materializes the "
-            "config as the non-root container user, can write it; k8s mounts it read-only there "
-            "via a ConfigMap subPath."
+            "deployments. Fabric deployments use agent.yaml in the same directory. Must be writable "
+            "by every runtime user this image can run as: the plain-docker container user and the "
+            "openshell sandbox user (uid 999, which cannot write the image's /workspace). /tmp is "
+            "the writable intersection; k8s mounts it read-only there via a ConfigMap subPath."
         ),
     )
 

@@ -14,11 +14,8 @@
 from __future__ import annotations
 
 import asyncio
-import base64
 import copy
-import json
 import logging
-import shlex
 import time
 from pathlib import PurePosixPath
 from typing import Any
@@ -67,10 +64,7 @@
 _PLUGIN_WHEELS_VOLUME = "plugin-wheels"
 _PLUGIN_WHEELS_MOUNT = "/opt/nemo/plugin-wheels"
 _NAT_CONFIG_ENV = "NAT_CONFIG_PATH"
-_NAT_CONFIG_YAML_ENV = "NAT_CONFIG_YAML"
-_AGENT_CONFIG_YAML_ENV = "AGENT_CONFIG_YAML"
 _AGENT_CONFIG_PATH_ENV = "AGENT_CONFIG_PATH"
-_STAGED_CONFIG_FILES_ENV = "STAGED_CONFIG_FILES_B64_JSON"
 _FABRIC_SERVER_MODULE = "nemo_agents_plugin.fabric.server"
 _AUTH_PROXY_IDENTITY = "agents"
 
@@ -256,29 +250,6 @@ def _fabric_server_cli_args(*, config_path: str, port: int) -> list[str]:
     ]
 
 
-def _materialize_config_and_exec(*, config_path: str, yaml_env: str, argv: list[str]) -> list[str]:
-    """Return ``sh -c`` args that write the config from *yaml_env*, then exec *argv*.
-
-    Docker mode needs this because the docker backend does not mount ``config_files``.
-    Paths and argv are shell-escaped so spaces/metacharacters cannot break the script.
-    """
-    quoted_path = shlex.quote(config_path)
-    quoted_argv = " ".join(shlex.quote(arg) for arg in argv)
-    return [f'mkdir -p "$(dirname {quoted_path})" && printf "%s" "${yaml_env}" > {quoted_path} && exec {quoted_argv}']
-
-
-def _materialize_staged_config_files_and_exec(*, env_name: str, argv: list[str]) -> list[str]:
-    """Return ``sh -c`` args that write staged ``config_files`` from *env_name*, then exec *argv*."""
-    quoted_argv = " ".join(shlex.quote(arg) for arg in argv)
-    inline_python = (
-        "import base64,json,os,pathlib;"
-        f"data=json.loads(os.environ[{json.dumps(env_name)}]);"
-        "[(pathlib.Path(p).parent.mkdir(parents=True,exist_ok=True),"
-        "pathlib.Path(p).write_bytes(base64.b64decode(b))) for p,b in data.items()]"
-    )
-    return [f"python -c {shlex.quote(inline_python)} && exec {quoted_argv}"]
-
-
 def executor_for_mode(config: DeploymentsRunnerConfig, mode: DeploymentMode) -> str | None:
     """Resolve the named deployments-plugin executor for *mode*."""
     if mode == "docker":
@@ -393,11 +364,9 @@ def build_deployment_config(
         env.append(EnvVar(name="PYTHONPATH", value=_PLUGIN_WHEELS_MOUNT))
 
     if is_fabric:
-        config_yaml_env = _AGENT_CONFIG_YAML_ENV
         server_command = ["python"]
         server_args = _fabric_server_cli_args(config_path=config_path, port=port)
     else:
-        config_yaml_env = _NAT_CONFIG_YAML_ENV
         server_command = ["nat", "start", "fastapi"]
         server_args = [
             "--config_file",
@@ -408,31 +377,8 @@ def build_deployment_config(
             str(port),
         ]
 
-    if mode == "docker":
-        # Docker backend does not mount config_files; materialize staged files from env.
-        if len(resolved_config_files) == 1:
-            single = resolved_config_files[0]
-            env.append(EnvVar(name=config_yaml_env, value=single.content))
-            command = ["sh", "-c"]
-            args = _materialize_config_and_exec(
-                config_path=single.path,
-                yaml_env=config_yaml_env,
-                argv=[*server_command, *server_args],
-            )
-        else:
-            payload = {
-                config_file.path: base64.b64encode(config_file.content.encode("utf-8")).decode("ascii")
-                for config_file in resolved_config_files
-            }
-            env.append(EnvVar(name=_STAGED_CONFIG_FILES_ENV, value=json.dumps(payload, separators=(",", ":"))))
-            command = ["sh", "-c"]
-            args = _materialize_staged_config_files_and_exec(
-                env_name=_STAGED_CONFIG_FILES_ENV,
-                argv=[*server_command, *server_args],
-            )
-    else:
-        command = server_command
-        args = server_args
+    command = server_command
+    args = server_args
 
     container = Container(
         name=_CONTAINER_NAME,

@@ -217,10 +217,7 @@ def test_executor_for_mode_prefers_mode_specific() -> None:
 
 
 def test_config_mount_path_default_is_under_writable_workspace() -> None:
-    # Docker mode materializes the config as the non-root container user, so the
-    # default must live under the image's writable WORKDIR (/workspace); a
-    # root-level path like /config is not writable and crash-loops the container.
-    assert DeploymentsRunnerConfig().config_mount_path.startswith("/workspace/")
+    assert DeploymentsRunnerConfig().config_mount_path.startswith("/tmp/nemo/")
 
 
 def test_build_deployment_config_always_single_container() -> None:
@@ -231,21 +228,20 @@ def test_build_deployment_config_always_single_container() -> None:
         port=8000,
         agent_config={"llms": {"nim": {"_type": "nim"}}},
         platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/config.yaml",
+        config_mount_path="/tmp/nemo/config.yaml",
         mode="docker",
     )
     assert cfg.restart_policy == "Always"
     assert len(cfg.containers) == 1
     container = cfg.containers[0]
     assert container.image == "nat-runtime:latest"
-    # Docker materializes config from NAT_CONFIG_YAML because config_files are not mounted.
-    assert container.command == ["sh", "-c"]
-    assert any(e.name == "NAT_CONFIG_YAML" for e in container.env)
+    assert container.command == ["nat", "start", "fastapi"]
+    assert not any(e.name == "NAT_CONFIG_YAML" for e in container.env)
     assert next(e.value for e in container.env if e.name == "NMP_BASE_URL") == "http://host.docker.internal:8080"
     assert container.readiness_probe is not None
     assert cfg.init_containers == []
     assert len(cfg.config_files) == 1
-    assert cfg.config_files[0].path == "/workspace/config.yaml"
+    assert cfg.config_files[0].path == "/tmp/nemo/config.yaml"
     loaded = yaml.safe_load(cfg.config_files[0].content)
     assert loaded["llms"]["nim"]["_type"] == "nim"
 
@@ -316,22 +312,6 @@ def test_build_deployment_config_docker_never_emits_init_containers() -> None:
 }
 
 
-def test_build_deployment_config_docker_shell_escapes_config_path() -> None:
-    cfg = build_deployment_config(
-        name="spaced-dep",
-        workspace="default",
-        image="nat-runtime:latest",
-        port=8000,
-        agent_config={"llms": {"nim": {"_type": "nim"}}},
-        platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/my config/config.yaml",
-        mode="docker",
-    )
-    script = cfg.containers[0].args[0]
-    assert "'/workspace/my config/config.yaml'" in script
-    assert 'printf "%s" "$NAT_CONFIG_YAML"' in script
-
-
 def test_build_deployment_config_fabric_docker_uses_fabric_server() -> None:
     cfg = build_deployment_config(
         name="fabric-dep",
@@ -340,23 +320,27 @@ def test_build_deployment_config_fabric_docker_uses_fabric_server() -> None:
         port=8000,
         agent_config=_FABRIC_AGENT_CONFIG,
         platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/config.yaml",
+        config_mount_path="/tmp/nemo/config.yaml",
         mode="docker",
     )
     container = cfg.containers[0]
-    assert container.command == ["sh", "-c"]
-    assert any(e.name == "AGENT_CONFIG_YAML" for e in container.env)
+    assert container.command == ["python"]
+    assert container.args[0] == "-m"
+    assert container.args[1] == "nemo_agents_plugin.fabric.server"
+    assert "--agent-config" in container.args
+    assert "/tmp/nemo/agent.yaml" in container.args
+    assert "--host" in container.args and "0.0.0.0" in container.args
+    assert not any(e.name == "AGENT_CONFIG_YAML" for e in container.env)
     assert not any(e.name == "NAT_CONFIG_YAML" for e in container.env)
-    assert any(e.name == "AGENT_CONFIG_PATH" and e.value == "/workspace/agent.yaml" for e in container.env)
+    assert any(e.name == "AGENT_CONFIG_PATH" and e.value == "/tmp/nemo/agent.yaml" for e in container.env)
     assert next(e.value for e in container.env if e.name == "NMP_BASE_URL") == "http://host.docker.internal:8080"
     assert next(e.value for e in container.env if e.name == PLATFORM_IGW_API_KEY_ENV) == (
         PLATFORM_IGW_API_KEY_PLACEHOLDER
     )
-    assert "nemo_agents_plugin.fabric.server" in container.args[0]
+    assert cfg.config_files[0].path == "/tmp/nemo/agent.yaml"
     assert container.readiness_probe is not None
     assert container.readiness_probe.http_get is not None
     assert container.readiness_probe.http_get.path == "/health"
-    assert cfg.config_files[0].path == "/workspace/agent.yaml"
 
 
 def test_build_deployment_config_fabric_k8s_uses_fabric_entrypoint() -> None:
@@ -414,11 +398,11 @@ def test_build_deployment_config_fabric_direct_endpoint_has_no_placeholder() ->
     assert not any(e.name in {PLATFORM_IGW_API_KEY_ENV, "OPENAI_API_KEY"} for e in cfg.containers[0].env)
 
 
-def test_build_deployment_config_fabric_docker_materializes_multiple_config_files() -> None:
+def test_build_deployment_config_fabric_docker_mounts_multiple_config_files() -> None:
     staged_files = [
-        ConfigFile(path="/workspace/agent.yaml", content="name: fabric-agent\n"),
-        ConfigFile(path="/workspace/skills/review/SKILL.md", content="# Review\n"),
-        ConfigFile(path="/workspace/prompts/system.md", content="You are helpful.\n"),
+        ConfigFile(path="/tmp/nemo/agent.yaml", content="name: fabric-agent\n"),
+        ConfigFile(path="/tmp/nemo/skills/review/SKILL.md", content="# Review\n"),
+        ConfigFile(path="/tmp/nemo/prompts/system.md", content="You are helpful.\n"),
     ]
     cfg = build_deployment_config(
         name="fabric-dep",
@@ -427,21 +411,18 @@ def test_build_deployment_config_fabric_docker_materializes_multiple_config_file
         port=8000,
         agent_config=_FABRIC_AGENT_CONFIG,
         platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/config.yaml",
+        config_mount_path="/tmp/nemo/config.yaml",
         mode="docker",
         config_files=staged_files,
     )
     container = cfg.containers[0]
-    assert container.command == ["sh", "-c"]
-    assert not any(e.name == "AGENT_CONFIG_YAML" for e in container.env)
-    assert any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in container.env)
-    assert "python -c" in container.args[0]
-    assert "nemo_agents_plugin.fabric.server" in container.args[0]
+    assert container.command == ["python"]
+    assert not any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in container.env)
     assert len(cfg.config_files) == 3
     assert {item.path for item in cfg.config_files} == {
-        "/workspace/agent.yaml",
-        "/workspace/skills/review/SKILL.md",
-        "/workspace/prompts/system.md",
+        "/tmp/nemo/agent.yaml",
+        "/tmp/nemo/skills/review/SKILL.md",
+        "/tmp/nemo/prompts/system.md",
     }
 
 
@@ -746,7 +727,8 @@ async def test_create_deployment_fabric_docker_rewrites_model_base_url() -> None
         "http://host.docker.internal:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1"
     )
     assert created_config.labels["nemo.agents/runtime"] == "fabric"
-    assert "nemo_agents_plugin.fabric.server" in created_config.containers[0].args[0]
+    assert created_config.containers[0].command == ["python"]
+    assert "nemo_agents_plugin.fabric.server" in created_config.containers[0].args
 
 
 @pytest.mark.asyncio
@@ -997,7 +979,7 @@ async def test_create_deployment_fabric_docker_stages_fileset_artifacts() -> Non
     mock_stage.assert_awaited_once()
     created_config = entities.create.await_args_list[0].args[0]
     assert len(created_config.config_files) == 2
-    assert any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in created_config.containers[0].env)
+    assert not any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in created_config.containers[0].env)
 
 
 @pytest.mark.asyncio

@@ -6,8 +6,10 @@
 from __future__ import annotations
 
 import asyncio
+import io
 import logging
 import os
+import tarfile
 from typing import TYPE_CHECKING, Any
 
 from nemo_deployments_plugin.backends.base import (
@@ -57,7 +59,7 @@
     managed_by_filter,
 )
 from nemo_deployments_plugin.constants import MANAGED_BY_LABEL
-from nemo_deployments_plugin.entities import Container, Deployment, DeploymentConfig
+from nemo_deployments_plugin.entities import ConfigFile, Container, Deployment, DeploymentConfig
 from nemo_deployments_plugin.secrets import SecretResolutionError, resolve_deployment_config_secrets
 from nemo_deployments_plugin.types import Endpoint, RestartPolicy
 from nemo_platform_plugin.capabilities import docker_from_env_kwargs, probe_docker
@@ -93,6 +95,30 @@ def _is_ngc_image(image: str) -> bool:
     return image == NGC_IMAGE_REGISTRY or image.startswith(f"{NGC_IMAGE_REGISTRY}/")
 
 
+def _config_files_tar(config_files: list[ConfigFile]) -> bytes:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w") as tar:
+        seen_dirs: set[str] = set()
+        for cf in config_files:
+            rel = cf.path.lstrip("/")
+            parts = rel.split("/")
+            for i in range(1, len(parts)):
+                d = "/".join(parts[:i])
+                if d in seen_dirs:
+                    continue
+                seen_dirs.add(d)
+                info = tarfile.TarInfo(name=d)
+                info.type = tarfile.DIRTYPE
+                info.mode = 0o755
+                tar.addfile(info)
+            data = cf.content.encode("utf-8")
+            info = tarfile.TarInfo(name=rel)
+            info.size = len(data)
+            info.mode = 0o644
+            tar.addfile(info, io.BytesIO(data))
+    return buf.getvalue()
+
+
 class DockerDeploymentBackend(DeploymentBackend):
     """Manage deployments and volumes as Docker containers and volumes."""
 
@@ -329,7 +355,9 @@ async def create_deployment(
                 network=f"container:{c_name}",
             )
             try:
-                await asyncio.to_thread(self._client.containers.run, **sidecar_run_kwargs)
+                sidecar_create_kwargs = {k: v for k, v in sidecar_run_kwargs.items() if k != "detach"}
+                sidecar_container = await asyncio.to_thread(self._client.containers.create, **sidecar_create_kwargs)
+                await asyncio.to_thread(sidecar_container.start)
             except Exception as exc:
                 logger.exception("Failed to start sidecar container %s", sidecar_name)
                 # Tear the whole group down so we don't leave a half-started deployment.
@@ -421,10 +449,15 @@ async def _run_server_container(
                 gpu_ids=gpu_ids,
                 network=network,
             )
+            create_kwargs = {k: v for k, v in run_kwargs.items() if k != "detach"}
             try:
-                container = await asyncio.to_thread(self._client.containers.run, **run_kwargs)
+                container = await asyncio.to_thread(self._client.containers.create, **create_kwargs)
+                if config.config_files:
+                    await self._deliver_config_files(container, config.config_files)
+                await asyncio.to_thread(container.start)
                 return container, host_ports, ""
             except Exception as exc:
+                await self._remove_container_by_name(name)
                 last_attempt = attempt == _PORT_CONFLICT_ATTEMPTS
                 if not host_ports or last_attempt or _PORT_CONFLICT_MARKER not in str(exc):
                     logger.exception("Failed to start container %s", name)
@@ -438,9 +471,6 @@ async def _run_server_container(
                     _PORT_CONFLICT_ATTEMPTS,
                     sorted(rejected_ports),
                 )
-                # containers.run() creates then starts, so a failed start leaves the
-                # created container holding the name and blocking the retry.
-                await self._remove_container_by_name(name)
                 try:
                     reallocated = await self._allocate_host_ports(container_spec, exclude_ports=rejected_ports)
                 except PortEnumerationError as port_exc:
@@ -449,6 +479,14 @@ async def _run_server_container(
                     return None, host_ports, "No host ports available in configured range"
                 host_ports = reallocated
 
+    async def _deliver_config_files(
+        self,
+        container: DockerContainer,
+        config_files: list[ConfigFile],
+    ) -> None:
+        archive = _config_files_tar(config_files)
+        await asyncio.to_thread(container.put_archive, "/", archive)
+
     def _build_run_kwargs(
         self,
         *,
@@ -553,7 +591,9 @@ async def _run_init_container(
             run_kwargs["volumes"] = volume_bindings
 
         def _run_and_wait() -> int:
-            container = self._client.containers.run(**run_kwargs)
+            create_kwargs = {k: v for k, v in run_kwargs.items() if k != "detach"}
+            container = self._client.containers.create(**create_kwargs)
+            container.start()
             result = container.wait(timeout=self._executor_config.docker_timeout)
             exit_code = self._exit_code_from_wait_result(result)
             try: