fix(docker): reduce runtime CVE exposure, fix e2e tests

docker-bake.hcl

@@ -32,6 +32,18 @@ variable "NMP_PYTHON_IMAGE" {
   default = "python:3.13.14-slim-trixie"
 }
 
+variable "DISTROLESS_BASE_3_13" {
+  default = "nvcr.io/nvidia/distroless/python:3.13-v4.0.9"
+}
+
+variable "NMP_API_RUNTIME_BASE" {
+  default = "root-distroless-base-3-13"
+}
+
+variable "NMP_CORE_RUNTIME_BASE" {
+  default = "root-distroless-base-3-13"
+}
+
 variable "AUTOMODEL_BASE_CONTEXT" {
   default = ""
 }
@@ -597,11 +609,14 @@ target "nmp-api-docker" {
     nmp-jobs-launcher         = "target:nmp-jobs-launcher"
     nmp-studio-ui             = "target:nmp-studio-ui"
     policy-wasm-artifacts     = "target:root-policy-wasm-artifacts"
+    root-busybox              = "target:root-busybox"
+    root-distroless-base-3-13 = "target:root-distroless-base-3-13"
     fastembed-cache           = FASTEMBED_CACHE_CONTEXT
   }
   args = {
     NMP_PLATFORM_VERSION = notequal(BAKE_TAG, "") ? BAKE_TAG : "dev"
     NMP_CODE_REVISION   = notequal(CI_COMMIT_SHA, "") ? CI_COMMIT_SHA : "dev"
+    NMP_API_RUNTIME_BASE = NMP_API_RUNTIME_BASE
   }
   cache-to   = maybe_registry_cache_to("nmp-api")
   cache-from = maybe_registry_cache_from("nmp-api")
@@ -620,6 +635,11 @@ target "nmp-core-docker" {
     nmp-workspace             = "target:nmp-workspace"
     nmp-jobs-launcher         = "target:nmp-jobs-launcher"
     policy-wasm-artifacts     = "target:root-policy-wasm-artifacts"
+    root-busybox              = "target:root-busybox"
+    root-distroless-base-3-13 = "target:root-distroless-base-3-13"
+  }
+  args = {
+    NMP_CORE_RUNTIME_BASE = NMP_CORE_RUNTIME_BASE
   }
   cache-to   = maybe_registry_cache_to("nmp-core")
   cache-from = maybe_registry_cache_from("nmp-core")
@@ -742,6 +762,16 @@ target "root-distroless-base-3-11" {
   }
 }
 
+target "root-distroless-base-3-13" {
+  target     = "root-distroless-base-3-13"
+  context    = "."
+  dockerfile = "docker/Dockerfile.bake"
+  platforms  = get_platforms()
+  args = {
+    DISTROLESS_BASE_3_13 = DISTROLESS_BASE_3_13
+  }
+}
+
 target "root-lib-source-artifacts" {
   target     = "root-lib-source-artifacts"
   context    = "."

docker/Dockerfile.auditor-tasks

@@ -68,7 +68,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --no-deps --python /app/.garak_venv/bin/python \
     "litellm>=1.83.0" \
     "urllib3>=2.6.3" \
-    "aiohttp>=3.13.3" \
+    "aiohttp>=3.14.3,<4" \
     "pyasn1>=0.6.4" \
     "protobuf>=6.33.5,<7.0.0" \
     "langchain-core>=1.2.22" \
@@ -81,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --no-deps \
     "urllib3>=2.6.3" \
     "protobuf>=5.29.6,<6.0.0" \
-    "aiohttp>=3.13.3" \
+    "aiohttp>=3.14.3,<4" \
     "pyasn1>=0.6.4" \
     "wheel>=0.46.2" \
     "orjson>=3.11.6" \

docker/Dockerfile.bake

@@ -3,6 +3,7 @@
 #######
 
 ARG DISTROLESS_BASE=nvcr.io/nvidia/distroless/python:3.11-v4.0.8
+ARG DISTROLESS_BASE_3_13=nvcr.io/nvidia/distroless/python:3.13-v4.0.9
 
 #root-uv-binary-base
     FROM ghcr.io/astral-sh/uv:0.9.14 AS root-uv-binary-base
@@ -54,6 +55,9 @@ ARG DISTROLESS_BASE=nvcr.io/nvidia/distroless/python:3.11-v4.0.8
 #root-distroless-base-3-11
     FROM ${DISTROLESS_BASE} AS root-distroless-base-3-11
 
+#root-distroless-base-3-13
+    FROM ${DISTROLESS_BASE_3_13} AS root-distroless-base-3-13
+
 #root-golang-base:
     FROM golang:1.23 AS root-golang-base
     RUN curl -sSfL https://github.com/golangci/golangci-lint/releases/download/v1.61.0/golangci-lint-1.61.0-linux-amd64.tar.gz -o golangci-lint.tar.gz \

docker/Dockerfile.nmp-api

@@ -2,6 +2,7 @@
 # NMP API - All Python services (core + application)
 
 ARG NMP_PYTHON_BASE=nmp-python-base
+ARG NMP_API_RUNTIME_BASE=nmp-python-base
 
 FROM ${NMP_PYTHON_BASE} AS builder
 COPY --from=nmp-workspace . .
@@ -54,30 +55,49 @@ RUN if find /tmp/fastembed_cache/models--qdrant--all-MiniLM-L6-v2-onnx/snapshots
         uv run --no-sync python -c 'import os, tempfile; from huggingface_hub import snapshot_download; snapshot_download(repo_id="qdrant/all-MiniLM-L6-v2-onnx", cache_dir=os.path.join(tempfile.gettempdir(), "fastembed_cache"))'; \
     fi
 
-FROM ${NMP_PYTHON_BASE} AS runtime
-ARG USERNAME=nvs
-ARG USER_UID=1000
-ARG USER_GID=1000
+FROM ${NMP_API_RUNTIME_BASE} AS runtime
 WORKDIR /app
-RUN groupadd --gid ${USER_GID} ${USERNAME} && \
-    useradd --uid ${USER_UID} --gid ${USER_GID} --create-home --shell /bin/bash ${USERNAME}
+
+# Local subprocess jobs need a small POSIX command surface; distroless does not
+# include these by default.
+COPY --from=root-busybox /bin/chmod /bin/chmod
+COPY --from=root-busybox /bin/sh /bin/sh
+COPY --from=root-busybox /bin/ls /bin/ls
+COPY --from=root-busybox /bin/mkdir /bin/mkdir
+COPY --from=root-busybox /bin/ln /bin/ln
+COPY --from=root-busybox /bin/rm /bin/rm
+COPY --from=root-busybox /bin/cp /bin/cp
+COPY --from=root-busybox /bin/wget /bin/wget
+COPY --from=root-busybox /bin/tar /bin/tar
+COPY --from=root-busybox /bin/mv /bin/mv
+COPY --from=root-busybox /bin/cat /bin/cat
+COPY --from=root-busybox /bin/head /bin/head
+COPY --from=root-busybox /bin/tail /bin/tail
+COPY --from=root-busybox /bin/grep /bin/grep
+COPY --from=root-busybox /bin/echo /bin/echo
+COPY --from=root-busybox /bin/sleep /bin/sleep
+COPY --from=root-busybox /bin/dirname /bin/dirname
+
 ENV USERNAME=nvs \
     HOME=/home/nvs
 # DuckDB extension cache (aws, httpfs) for S3 log storage
-COPY --chown=nvs:nvs --from=builder /root/.duckdb /root/.duckdb
-COPY --chown=nvs:nvs --from=builder /root/.duckdb /home/nvs/.duckdb
-COPY --chown=nvs:nvs --from=builder /app/.venv /app/.venv
-ENV PATH="/app/.venv/bin:$PATH"
-COPY --chown=nvs:nvs --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher
-COPY --chown=nvs:nvs --from=nmp-studio-ui /artifacts /static/studio
-COPY --chown=nvs:nvs --from=builder /app/plugins/nemo-data-designer/tiktoken-cache /app/tiktoken-cache
+COPY --chown=1000:1000 --from=builder /root/.duckdb /home/nvs/.duckdb
+COPY --chown=1000:1000 --from=builder /app/.venv /app/.venv
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH" \
+    PYTHONPATH="/app/.venv/lib/python${PY_VERSION}/site-packages"
+COPY --chown=1000:1000 --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher
+COPY --chown=1000:1000 --from=nmp-studio-ui /artifacts /static/studio
+COPY --chown=1000:1000 --from=builder /app/plugins/nemo-data-designer/tiktoken-cache /app/tiktoken-cache
 ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache
-COPY --chown=nvs:nvs --from=builder /tmp/fastembed_cache /tmp/fastembed_cache
+COPY --chown=1000:1000 --from=builder /tmp/fastembed_cache /tmp/fastembed_cache
 ARG NMP_PLATFORM_VERSION=dev
 ARG NMP_CODE_REVISION=
 ENV NMP_PLATFORM_VERSION=${NMP_PLATFORM_VERSION}
 ENV NMP_CODE_REVISION=${NMP_CODE_REVISION}
 EXPOSE 8080
+# Keep runtime UID aligned with the previous API image until Docker/K8s
+# storage mounts are consistently writable by the non-root nvs user.
 USER 0
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
     CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"]

docker/Dockerfile.nmp-core

@@ -2,6 +2,7 @@
 # NMP Core - Minimal core infrastructure APIs
 
 ARG NMP_PYTHON_BASE=nmp-python-base
+ARG NMP_CORE_RUNTIME_BASE=nmp-python-base
 
 FROM ${NMP_PYTHON_BASE} AS builder
 COPY --from=nmp-workspace . .
@@ -18,21 +19,39 @@ RUN /app/.venv/bin/python -c 'from nmp.core.auth.app.embedded_pdp.policy_wasm im
 # because DuckDB's default remote INSTALL path can be blocked or flaky in some build environments.
 RUN sh /app/script/install_duckdb_extensions.sh
 
-FROM ${NMP_PYTHON_BASE} AS runtime
-ARG USERNAME=nvs
-ARG USER_UID=1000
-ARG USER_GID=1000
+FROM ${NMP_CORE_RUNTIME_BASE} AS runtime
 WORKDIR /app
-RUN groupadd --gid ${USER_GID} ${USERNAME} && \
-    useradd --uid ${USER_UID} --gid ${USER_GID} --create-home --shell /bin/bash ${USERNAME}
+
+# Local subprocess jobs need a small POSIX command surface; distroless does not
+# include these by default.
+COPY --from=root-busybox /bin/chmod /bin/chmod
+COPY --from=root-busybox /bin/sh /bin/sh
+COPY --from=root-busybox /bin/ls /bin/ls
+COPY --from=root-busybox /bin/mkdir /bin/mkdir
+COPY --from=root-busybox /bin/ln /bin/ln
+COPY --from=root-busybox /bin/rm /bin/rm
+COPY --from=root-busybox /bin/cp /bin/cp
+COPY --from=root-busybox /bin/wget /bin/wget
+COPY --from=root-busybox /bin/tar /bin/tar
+COPY --from=root-busybox /bin/mv /bin/mv
+COPY --from=root-busybox /bin/cat /bin/cat
+COPY --from=root-busybox /bin/head /bin/head
+COPY --from=root-busybox /bin/tail /bin/tail
+COPY --from=root-busybox /bin/grep /bin/grep
+COPY --from=root-busybox /bin/echo /bin/echo
+COPY --from=root-busybox /bin/sleep /bin/sleep
+COPY --from=root-busybox /bin/dirname /bin/dirname
+
 ENV USERNAME=nvs \
     HOME=/home/nvs
 # DuckDB extension cache (aws, httpfs) for S3 log storage
-COPY --chown=nvs:nvs --from=builder /root/.duckdb /root/.duckdb
-COPY --chown=nvs:nvs --from=builder /root/.duckdb /home/nvs/.duckdb
-COPY --chown=nvs:nvs --from=builder /app/.venv /app/.venv
-ENV PATH="/app/.venv/bin:$PATH"
-COPY --chown=nvs:nvs --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher
+COPY --chown=1000:1000 --from=builder /root/.duckdb /root/.duckdb
+COPY --chown=1000:1000 --from=builder /root/.duckdb /home/nvs/.duckdb
+COPY --chown=1000:1000 --from=builder /app/.venv /app/.venv
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH" \
+    PYTHONPATH="/app/.venv/lib/python${PY_VERSION}/site-packages"
+COPY --chown=1000:1000 --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher
 ENTRYPOINT ["nemo", "services", "run"]
 EXPOSE 8080
 USER 0

docker/Dockerfile.nmp-unsloth-training

@@ -174,7 +174,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache \
     --overrides /opt/docker/preserve_base_torch.txt \
-    "aiohttp>=3.13.4,<4" \
+    "aiohttp>=3.14.3,<4" \
     "urllib3>=2.7.0,<3" \
     "grpcio>=1.81.1,<2" \
     "mistune>=3.3.3,<4" \
@@ -208,6 +208,11 @@ RUN rm -rf \
     /usr/local/lib/python3.12/dist-packages/mlflow \
     /usr/local/lib/python3.12/dist-packages/mlflow-*.dist-info \
     /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info \
+    /usr/local/lib/python3.12/dist-packages/git \
+    /usr/local/lib/python3.12/dist-packages/gitpython-*.dist-info \
+    /usr/local/lib/python3.12/dist-packages/GitPython-*.dist-info \
+    /usr/local/lib/python3.12/dist-packages/jupyterlab \
+    /usr/local/lib/python3.12/dist-packages/jupyterlab-*.dist-info \
     /usr/local/lib/python3.12/dist-packages/PIL \
     /usr/local/lib/python3.12/dist-packages/pillow.libs \
     /usr/local/lib/python3.12/dist-packages/pillow-*.dist-info \

docker/Dockerfile.safe-synthesizer-tasks

@@ -116,6 +116,7 @@ RUN printf '%s\n' \
         > /tmp/safe-synthesizer-runtime.txt && \
     printf '%s\n' \
         wandb==0.28.1 \
+        'aiohttp>=3.14.3,<4' \
         'cryptography>=50.0.0,<51' \
         'pyarrow>=23.0.1,<24' \
         > /tmp/safe-synthesizer-overrides.txt

e2e/k8s/values/kind.yaml

@@ -7,6 +7,14 @@ postgresql:
   image:
     repository: docker.io/library/postgres
 
+clickhouse:
+  persistence:
+    enabled: false
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+
 core:
   storage:
     accessModes: