fix(deployments): docker backend delivers config_files; unify agents config delivery

plugins/nemo-agents/src/nemo_agents_plugin/config.py

@@ -98,13 +98,13 @@ class DeploymentsRunnerConfig(BaseModel):
         ),
     )
     config_mount_path: str = Field(
-        default="/workspace/config.yaml",
+        default="/tmp/nemo/config.yaml",
         description=(
             "Path inside the container where the NAT workflow config is placed for nat-workflow-v1 "
-            "deployments. Fabric deployments use agent.yaml in the same directory. Must sit under "
-            "the image's writable WORKDIR (/workspace) so docker mode, which materializes the "
-            "config as the non-root container user, can write it; k8s mounts it read-only there "
-            "via a ConfigMap subPath."
+            "deployments. Fabric deployments use agent.yaml in the same directory. Must be writable "
+            "by every runtime user this image can run as: the plain-docker container user and the "
+            "openshell sandbox user (uid 999, which cannot write the image's /workspace). /tmp is "
+            "the writable intersection; k8s mounts it read-only there via a ConfigMap subPath."
         ),
     )
 

plugins/nemo-agents/src/nemo_agents_plugin/runner/deployments_backend.py

@@ -14,11 +14,8 @@
 from __future__ import annotations
 
 import asyncio
-import base64
 import copy
-import json
 import logging
-import shlex
 import time
 from pathlib import PurePosixPath
 from typing import Any
@@ -67,10 +64,7 @@
 _PLUGIN_WHEELS_VOLUME = "plugin-wheels"
 _PLUGIN_WHEELS_MOUNT = "/opt/nemo/plugin-wheels"
 _NAT_CONFIG_ENV = "NAT_CONFIG_PATH"
-_NAT_CONFIG_YAML_ENV = "NAT_CONFIG_YAML"
-_AGENT_CONFIG_YAML_ENV = "AGENT_CONFIG_YAML"
 _AGENT_CONFIG_PATH_ENV = "AGENT_CONFIG_PATH"
-_STAGED_CONFIG_FILES_ENV = "STAGED_CONFIG_FILES_B64_JSON"
 _FABRIC_SERVER_MODULE = "nemo_agents_plugin.fabric.server"
 _AUTH_PROXY_IDENTITY = "agents"
 
@@ -256,29 +250,6 @@ def _fabric_server_cli_args(*, config_path: str, port: int) -> list[str]:
     ]
 
 
-def _materialize_config_and_exec(*, config_path: str, yaml_env: str, argv: list[str]) -> list[str]:
-    """Return ``sh -c`` args that write the config from *yaml_env*, then exec *argv*.
-
-    Docker mode needs this because the docker backend does not mount ``config_files``.
-    Paths and argv are shell-escaped so spaces/metacharacters cannot break the script.
-    """
-    quoted_path = shlex.quote(config_path)
-    quoted_argv = " ".join(shlex.quote(arg) for arg in argv)
-    return [f'mkdir -p "$(dirname {quoted_path})" && printf "%s" "${yaml_env}" > {quoted_path} && exec {quoted_argv}']
-
-
-def _materialize_staged_config_files_and_exec(*, env_name: str, argv: list[str]) -> list[str]:
-    """Return ``sh -c`` args that write staged ``config_files`` from *env_name*, then exec *argv*."""
-    quoted_argv = " ".join(shlex.quote(arg) for arg in argv)
-    inline_python = (
-        "import base64,json,os,pathlib;"
-        f"data=json.loads(os.environ[{json.dumps(env_name)}]);"
-        "[(pathlib.Path(p).parent.mkdir(parents=True,exist_ok=True),"
-        "pathlib.Path(p).write_bytes(base64.b64decode(b))) for p,b in data.items()]"
-    )
-    return [f"python -c {shlex.quote(inline_python)} && exec {quoted_argv}"]
-
-
 def executor_for_mode(config: DeploymentsRunnerConfig, mode: DeploymentMode) -> str | None:
     """Resolve the named deployments-plugin executor for *mode*."""
     if mode == "docker":
@@ -393,11 +364,9 @@ def build_deployment_config(
         env.append(EnvVar(name="PYTHONPATH", value=_PLUGIN_WHEELS_MOUNT))
 
     if is_fabric:
-        config_yaml_env = _AGENT_CONFIG_YAML_ENV
         server_command = ["python"]
         server_args = _fabric_server_cli_args(config_path=config_path, port=port)
     else:
-        config_yaml_env = _NAT_CONFIG_YAML_ENV
         server_command = ["nat", "start", "fastapi"]
         server_args = [
             "--config_file",
@@ -408,31 +377,8 @@ def build_deployment_config(
             str(port),
         ]
 
-    if mode == "docker":
-        # Docker backend does not mount config_files; materialize staged files from env.
-        if len(resolved_config_files) == 1:
-            single = resolved_config_files[0]
-            env.append(EnvVar(name=config_yaml_env, value=single.content))
-            command = ["sh", "-c"]
-            args = _materialize_config_and_exec(
-                config_path=single.path,
-                yaml_env=config_yaml_env,
-                argv=[*server_command, *server_args],
-            )
-        else:
-            payload = {
-                config_file.path: base64.b64encode(config_file.content.encode("utf-8")).decode("ascii")
-                for config_file in resolved_config_files
-            }
-            env.append(EnvVar(name=_STAGED_CONFIG_FILES_ENV, value=json.dumps(payload, separators=(",", ":"))))
-            command = ["sh", "-c"]
-            args = _materialize_staged_config_files_and_exec(
-                env_name=_STAGED_CONFIG_FILES_ENV,
-                argv=[*server_command, *server_args],
-            )
-    else:
-        command = server_command
-        args = server_args
+    command = server_command
+    args = server_args
 
     container = Container(
         name=_CONTAINER_NAME,

plugins/nemo-agents/tests/unit/test_runner_deployments.py

@@ -217,10 +217,7 @@ def test_executor_for_mode_prefers_mode_specific() -> None:
 
 
 def test_config_mount_path_default_is_under_writable_workspace() -> None:
-    # Docker mode materializes the config as the non-root container user, so the
-    # default must live under the image's writable WORKDIR (/workspace); a
-    # root-level path like /config is not writable and crash-loops the container.
-    assert DeploymentsRunnerConfig().config_mount_path.startswith("/workspace/")
+    assert DeploymentsRunnerConfig().config_mount_path == "/tmp/nemo/config.yaml"
 
 
 def test_build_deployment_config_always_single_container() -> None:
@@ -231,21 +228,20 @@ def test_build_deployment_config_always_single_container() -> None:
         port=8000,
         agent_config={"llms": {"nim": {"_type": "nim"}}},
         platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/config.yaml",
+        config_mount_path="/tmp/nemo/config.yaml",
         mode="docker",
     )
     assert cfg.restart_policy == "Always"
     assert len(cfg.containers) == 1
     container = cfg.containers[0]
     assert container.image == "nat-runtime:latest"
-    # Docker materializes config from NAT_CONFIG_YAML because config_files are not mounted.
-    assert container.command == ["sh", "-c"]
-    assert any(e.name == "NAT_CONFIG_YAML" for e in container.env)
+    assert container.command == ["nat", "start", "fastapi"]
+    assert not any(e.name == "NAT_CONFIG_YAML" for e in container.env)
     assert next(e.value for e in container.env if e.name == "NMP_BASE_URL") == "http://host.docker.internal:8080"
     assert container.readiness_probe is not None
     assert cfg.init_containers == []
     assert len(cfg.config_files) == 1
-    assert cfg.config_files[0].path == "/workspace/config.yaml"
+    assert cfg.config_files[0].path == "/tmp/nemo/config.yaml"
     loaded = yaml.safe_load(cfg.config_files[0].content)
     assert loaded["llms"]["nim"]["_type"] == "nim"
 
@@ -316,22 +312,6 @@ def test_build_deployment_config_docker_never_emits_init_containers() -> None:
 }
 
 
-def test_build_deployment_config_docker_shell_escapes_config_path() -> None:
-    cfg = build_deployment_config(
-        name="spaced-dep",
-        workspace="default",
-        image="nat-runtime:latest",
-        port=8000,
-        agent_config={"llms": {"nim": {"_type": "nim"}}},
-        platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/my config/config.yaml",
-        mode="docker",
-    )
-    script = cfg.containers[0].args[0]
-    assert "'/workspace/my config/config.yaml'" in script
-    assert 'printf "%s" "$NAT_CONFIG_YAML"' in script
-
-
 def test_build_deployment_config_fabric_docker_uses_fabric_server() -> None:
     cfg = build_deployment_config(
         name="fabric-dep",
@@ -340,23 +320,27 @@ def test_build_deployment_config_fabric_docker_uses_fabric_server() -> None:
         port=8000,
         agent_config=_FABRIC_AGENT_CONFIG,
         platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/config.yaml",
+        config_mount_path="/tmp/nemo/config.yaml",
         mode="docker",
     )
     container = cfg.containers[0]
-    assert container.command == ["sh", "-c"]
-    assert any(e.name == "AGENT_CONFIG_YAML" for e in container.env)
+    assert container.command == ["python"]
+    assert container.args[0] == "-m"
+    assert container.args[1] == "nemo_agents_plugin.fabric.server"
+    assert "--agent-config" in container.args
+    assert "/tmp/nemo/agent.yaml" in container.args
+    assert "--host" in container.args and "0.0.0.0" in container.args
+    assert not any(e.name == "AGENT_CONFIG_YAML" for e in container.env)
     assert not any(e.name == "NAT_CONFIG_YAML" for e in container.env)
-    assert any(e.name == "AGENT_CONFIG_PATH" and e.value == "/workspace/agent.yaml" for e in container.env)
+    assert any(e.name == "AGENT_CONFIG_PATH" and e.value == "/tmp/nemo/agent.yaml" for e in container.env)
     assert next(e.value for e in container.env if e.name == "NMP_BASE_URL") == "http://host.docker.internal:8080"
     assert next(e.value for e in container.env if e.name == PLATFORM_IGW_API_KEY_ENV) == (
         PLATFORM_IGW_API_KEY_PLACEHOLDER
     )
-    assert "nemo_agents_plugin.fabric.server" in container.args[0]
+    assert cfg.config_files[0].path == "/tmp/nemo/agent.yaml"
     assert container.readiness_probe is not None
     assert container.readiness_probe.http_get is not None
     assert container.readiness_probe.http_get.path == "/health"
-    assert cfg.config_files[0].path == "/workspace/agent.yaml"
 
 
 def test_build_deployment_config_fabric_k8s_uses_fabric_entrypoint() -> None:
@@ -414,11 +398,11 @@ def test_build_deployment_config_fabric_direct_endpoint_has_no_placeholder() ->
     assert not any(e.name in {PLATFORM_IGW_API_KEY_ENV, "OPENAI_API_KEY"} for e in cfg.containers[0].env)
 
 
-def test_build_deployment_config_fabric_docker_materializes_multiple_config_files() -> None:
+def test_build_deployment_config_fabric_docker_mounts_multiple_config_files() -> None:
     staged_files = [
-        ConfigFile(path="/workspace/agent.yaml", content="name: fabric-agent\n"),
-        ConfigFile(path="/workspace/skills/review/SKILL.md", content="# Review\n"),
-        ConfigFile(path="/workspace/prompts/system.md", content="You are helpful.\n"),
+        ConfigFile(path="/tmp/nemo/agent.yaml", content="name: fabric-agent\n"),
+        ConfigFile(path="/tmp/nemo/skills/review/SKILL.md", content="# Review\n"),
+        ConfigFile(path="/tmp/nemo/prompts/system.md", content="You are helpful.\n"),
     ]
     cfg = build_deployment_config(
         name="fabric-dep",
@@ -427,21 +411,18 @@ def test_build_deployment_config_fabric_docker_materializes_multiple_config_file
         port=8000,
         agent_config=_FABRIC_AGENT_CONFIG,
         platform_base_url="http://host.docker.internal:8080",
-        config_mount_path="/workspace/config.yaml",
+        config_mount_path="/tmp/nemo/config.yaml",
         mode="docker",
         config_files=staged_files,
     )
     container = cfg.containers[0]
-    assert container.command == ["sh", "-c"]
-    assert not any(e.name == "AGENT_CONFIG_YAML" for e in container.env)
-    assert any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in container.env)
-    assert "python -c" in container.args[0]
-    assert "nemo_agents_plugin.fabric.server" in container.args[0]
+    assert container.command == ["python"]
+    assert not any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in container.env)
     assert len(cfg.config_files) == 3
     assert {item.path for item in cfg.config_files} == {
-        "/workspace/agent.yaml",
-        "/workspace/skills/review/SKILL.md",
-        "/workspace/prompts/system.md",
+        "/tmp/nemo/agent.yaml",
+        "/tmp/nemo/skills/review/SKILL.md",
+        "/tmp/nemo/prompts/system.md",
     }
 
 
@@ -746,7 +727,8 @@ async def test_create_deployment_fabric_docker_rewrites_model_base_url() -> None
         "http://host.docker.internal:8080/apis/inference-gateway/v2/workspaces/default/openai/-/v1"
     )
     assert created_config.labels["nemo.agents/runtime"] == "fabric"
-    assert "nemo_agents_plugin.fabric.server" in created_config.containers[0].args[0]
+    assert created_config.containers[0].command == ["python"]
+    assert "nemo_agents_plugin.fabric.server" in created_config.containers[0].args
 
 
 @pytest.mark.asyncio
@@ -967,8 +949,8 @@ async def test_create_deployment_fabric_docker_stages_fileset_artifacts() -> Non
         "harnesses": {"main": {"kind": "codex", "settings": {}}},
     }
     staged_files = [
-        ConfigFile(path="/workspace/agent.yaml", content=yaml.safe_dump(config, sort_keys=False)),
-        ConfigFile(path="/workspace/skills/review/SKILL.md", content="# Review\n"),
+        ConfigFile(path="/tmp/nemo/agent.yaml", content=yaml.safe_dump(config, sort_keys=False)),
+        ConfigFile(path="/tmp/nemo/skills/review/SKILL.md", content="# Review\n"),
     ]
     sdk = MagicMock()
 
@@ -997,7 +979,7 @@ async def test_create_deployment_fabric_docker_stages_fileset_artifacts() -> Non
     mock_stage.assert_awaited_once()
     created_config = entities.create.await_args_list[0].args[0]
     assert len(created_config.config_files) == 2
-    assert any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in created_config.containers[0].env)
+    assert not any(e.name == "STAGED_CONFIG_FILES_B64_JSON" for e in created_config.containers[0].env)
 
 
 @pytest.mark.asyncio
@@ -1017,8 +999,8 @@ async def test_create_deployment_fabric_k8s_stages_fileset_artifacts() -> None:
         "harnesses": {"main": {"kind": "codex", "settings": {}}},
     }
     staged_files = [
-        ConfigFile(path="/workspace/agent.yaml", content=yaml.safe_dump(config, sort_keys=False)),
-        ConfigFile(path="/workspace/skills/review/SKILL.md", content="# Review\n"),
+        ConfigFile(path="/tmp/nemo/agent.yaml", content=yaml.safe_dump(config, sort_keys=False)),
+        ConfigFile(path="/tmp/nemo/skills/review/SKILL.md", content="# Review\n"),
     ]
     sdk = MagicMock()