diff --git a/docker-bake.hcl b/docker-bake.hcl index c4dbd5843b..1ea7a11246 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -32,6 +32,18 @@ variable "NMP_PYTHON_IMAGE" { default = "python:3.13.14-slim-trixie" } +variable "DISTROLESS_BASE_3_13" { + default = "nvcr.io/nvidia/distroless/python:3.13-v4.0.9" +} + +variable "NMP_API_RUNTIME_BASE" { + default = "root-distroless-base-3-13" +} + +variable "NMP_CORE_RUNTIME_BASE" { + default = "root-distroless-base-3-13" +} + variable "AUTOMODEL_BASE_CONTEXT" { default = "" } @@ -597,11 +609,14 @@ target "nmp-api-docker" { nmp-jobs-launcher = "target:nmp-jobs-launcher" nmp-studio-ui = "target:nmp-studio-ui" policy-wasm-artifacts = "target:root-policy-wasm-artifacts" + root-busybox = "target:root-busybox" + root-distroless-base-3-13 = "target:root-distroless-base-3-13" fastembed-cache = FASTEMBED_CACHE_CONTEXT } args = { NMP_PLATFORM_VERSION = notequal(BAKE_TAG, "") ? BAKE_TAG : "dev" NMP_CODE_REVISION = notequal(CI_COMMIT_SHA, "") ? CI_COMMIT_SHA : "dev" + NMP_API_RUNTIME_BASE = NMP_API_RUNTIME_BASE } cache-to = maybe_registry_cache_to("nmp-api") cache-from = maybe_registry_cache_from("nmp-api") @@ -620,6 +635,11 @@ target "nmp-core-docker" { nmp-workspace = "target:nmp-workspace" nmp-jobs-launcher = "target:nmp-jobs-launcher" policy-wasm-artifacts = "target:root-policy-wasm-artifacts" + root-busybox = "target:root-busybox" + root-distroless-base-3-13 = "target:root-distroless-base-3-13" + } + args = { + NMP_CORE_RUNTIME_BASE = NMP_CORE_RUNTIME_BASE } cache-to = maybe_registry_cache_to("nmp-core") cache-from = maybe_registry_cache_from("nmp-core") @@ -742,6 +762,16 @@ target "root-distroless-base-3-11" { } } +target "root-distroless-base-3-13" { + target = "root-distroless-base-3-13" + context = "." + dockerfile = "docker/Dockerfile.bake" + platforms = get_platforms() + args = { + DISTROLESS_BASE_3_13 = DISTROLESS_BASE_3_13 + } +} + target "root-lib-source-artifacts" { target = "root-lib-source-artifacts" context = "." diff --git a/docker/Dockerfile.auditor-tasks b/docker/Dockerfile.auditor-tasks index 77d61ee389..5eb62896b7 100644 --- a/docker/Dockerfile.auditor-tasks +++ b/docker/Dockerfile.auditor-tasks @@ -68,7 +68,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --no-deps --python /app/.garak_venv/bin/python \ "litellm>=1.83.0" \ "urllib3>=2.6.3" \ - "aiohttp>=3.13.3" \ + "aiohttp>=3.14.3,<4" \ "pyasn1>=0.6.4" \ "protobuf>=6.33.5,<7.0.0" \ "langchain-core>=1.2.22" \ @@ -81,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --no-deps \ "urllib3>=2.6.3" \ "protobuf>=5.29.6,<6.0.0" \ - "aiohttp>=3.13.3" \ + "aiohttp>=3.14.3,<4" \ "pyasn1>=0.6.4" \ "wheel>=0.46.2" \ "orjson>=3.11.6" \ diff --git a/docker/Dockerfile.bake b/docker/Dockerfile.bake index 03ba821ee2..0fe7494ed0 100644 --- a/docker/Dockerfile.bake +++ b/docker/Dockerfile.bake @@ -3,6 +3,7 @@ ####### ARG DISTROLESS_BASE=nvcr.io/nvidia/distroless/python:3.11-v4.0.8 +ARG DISTROLESS_BASE_3_13=nvcr.io/nvidia/distroless/python:3.13-v4.0.9 #root-uv-binary-base FROM ghcr.io/astral-sh/uv:0.9.14 AS root-uv-binary-base @@ -54,6 +55,9 @@ ARG DISTROLESS_BASE=nvcr.io/nvidia/distroless/python:3.11-v4.0.8 #root-distroless-base-3-11 FROM ${DISTROLESS_BASE} AS root-distroless-base-3-11 +#root-distroless-base-3-13 + FROM ${DISTROLESS_BASE_3_13} AS root-distroless-base-3-13 + #root-golang-base: FROM golang:1.23 AS root-golang-base RUN curl -sSfL https://github.com/golangci/golangci-lint/releases/download/v1.61.0/golangci-lint-1.61.0-linux-amd64.tar.gz -o golangci-lint.tar.gz \ diff --git a/docker/Dockerfile.nmp-api b/docker/Dockerfile.nmp-api index bb1f177dc2..48a3cb8ddc 100644 --- a/docker/Dockerfile.nmp-api +++ b/docker/Dockerfile.nmp-api @@ -2,6 +2,7 @@ # NMP API - All Python services (core + application) ARG NMP_PYTHON_BASE=nmp-python-base +ARG NMP_API_RUNTIME_BASE=nmp-python-base FROM ${NMP_PYTHON_BASE} AS builder COPY --from=nmp-workspace . . @@ -54,30 +55,49 @@ RUN if find /tmp/fastembed_cache/models--qdrant--all-MiniLM-L6-v2-onnx/snapshots uv run --no-sync python -c 'import os, tempfile; from huggingface_hub import snapshot_download; snapshot_download(repo_id="qdrant/all-MiniLM-L6-v2-onnx", cache_dir=os.path.join(tempfile.gettempdir(), "fastembed_cache"))'; \ fi -FROM ${NMP_PYTHON_BASE} AS runtime -ARG USERNAME=nvs -ARG USER_UID=1000 -ARG USER_GID=1000 +FROM ${NMP_API_RUNTIME_BASE} AS runtime WORKDIR /app -RUN groupadd --gid ${USER_GID} ${USERNAME} && \ - useradd --uid ${USER_UID} --gid ${USER_GID} --create-home --shell /bin/bash ${USERNAME} + +# Local subprocess jobs need a small POSIX command surface; distroless does not +# include these by default. +COPY --from=root-busybox /bin/chmod /bin/chmod +COPY --from=root-busybox /bin/sh /bin/sh +COPY --from=root-busybox /bin/ls /bin/ls +COPY --from=root-busybox /bin/mkdir /bin/mkdir +COPY --from=root-busybox /bin/ln /bin/ln +COPY --from=root-busybox /bin/rm /bin/rm +COPY --from=root-busybox /bin/cp /bin/cp +COPY --from=root-busybox /bin/wget /bin/wget +COPY --from=root-busybox /bin/tar /bin/tar +COPY --from=root-busybox /bin/mv /bin/mv +COPY --from=root-busybox /bin/cat /bin/cat +COPY --from=root-busybox /bin/head /bin/head +COPY --from=root-busybox /bin/tail /bin/tail +COPY --from=root-busybox /bin/grep /bin/grep +COPY --from=root-busybox /bin/echo /bin/echo +COPY --from=root-busybox /bin/sleep /bin/sleep +COPY --from=root-busybox /bin/dirname /bin/dirname + ENV USERNAME=nvs \ HOME=/home/nvs # DuckDB extension cache (aws, httpfs) for S3 log storage -COPY --chown=nvs:nvs --from=builder /root/.duckdb /root/.duckdb -COPY --chown=nvs:nvs --from=builder /root/.duckdb /home/nvs/.duckdb -COPY --chown=nvs:nvs --from=builder /app/.venv /app/.venv -ENV PATH="/app/.venv/bin:$PATH" -COPY --chown=nvs:nvs --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher -COPY --chown=nvs:nvs --from=nmp-studio-ui /artifacts /static/studio -COPY --chown=nvs:nvs --from=builder /app/plugins/nemo-data-designer/tiktoken-cache /app/tiktoken-cache +COPY --chown=1000:1000 --from=builder /root/.duckdb /home/nvs/.duckdb +COPY --chown=1000:1000 --from=builder /app/.venv /app/.venv +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" \ + PYTHONPATH="/app/.venv/lib/python${PY_VERSION}/site-packages" +COPY --chown=1000:1000 --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher +COPY --chown=1000:1000 --from=nmp-studio-ui /artifacts /static/studio +COPY --chown=1000:1000 --from=builder /app/plugins/nemo-data-designer/tiktoken-cache /app/tiktoken-cache ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache -COPY --chown=nvs:nvs --from=builder /tmp/fastembed_cache /tmp/fastembed_cache +COPY --chown=1000:1000 --from=builder /tmp/fastembed_cache /tmp/fastembed_cache ARG NMP_PLATFORM_VERSION=dev ARG NMP_CODE_REVISION= ENV NMP_PLATFORM_VERSION=${NMP_PLATFORM_VERSION} ENV NMP_CODE_REVISION=${NMP_CODE_REVISION} EXPOSE 8080 +# Keep runtime UID aligned with the previous API image until Docker/K8s +# storage mounts are consistently writable by the non-root nvs user. USER 0 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] diff --git a/docker/Dockerfile.nmp-core b/docker/Dockerfile.nmp-core index c9b73e35f3..3f8f876d03 100644 --- a/docker/Dockerfile.nmp-core +++ b/docker/Dockerfile.nmp-core @@ -2,6 +2,7 @@ # NMP Core - Minimal core infrastructure APIs ARG NMP_PYTHON_BASE=nmp-python-base +ARG NMP_CORE_RUNTIME_BASE=nmp-python-base FROM ${NMP_PYTHON_BASE} AS builder COPY --from=nmp-workspace . . @@ -18,21 +19,39 @@ RUN /app/.venv/bin/python -c 'from nmp.core.auth.app.embedded_pdp.policy_wasm im # because DuckDB's default remote INSTALL path can be blocked or flaky in some build environments. RUN sh /app/script/install_duckdb_extensions.sh -FROM ${NMP_PYTHON_BASE} AS runtime -ARG USERNAME=nvs -ARG USER_UID=1000 -ARG USER_GID=1000 +FROM ${NMP_CORE_RUNTIME_BASE} AS runtime WORKDIR /app -RUN groupadd --gid ${USER_GID} ${USERNAME} && \ - useradd --uid ${USER_UID} --gid ${USER_GID} --create-home --shell /bin/bash ${USERNAME} + +# Local subprocess jobs need a small POSIX command surface; distroless does not +# include these by default. +COPY --from=root-busybox /bin/chmod /bin/chmod +COPY --from=root-busybox /bin/sh /bin/sh +COPY --from=root-busybox /bin/ls /bin/ls +COPY --from=root-busybox /bin/mkdir /bin/mkdir +COPY --from=root-busybox /bin/ln /bin/ln +COPY --from=root-busybox /bin/rm /bin/rm +COPY --from=root-busybox /bin/cp /bin/cp +COPY --from=root-busybox /bin/wget /bin/wget +COPY --from=root-busybox /bin/tar /bin/tar +COPY --from=root-busybox /bin/mv /bin/mv +COPY --from=root-busybox /bin/cat /bin/cat +COPY --from=root-busybox /bin/head /bin/head +COPY --from=root-busybox /bin/tail /bin/tail +COPY --from=root-busybox /bin/grep /bin/grep +COPY --from=root-busybox /bin/echo /bin/echo +COPY --from=root-busybox /bin/sleep /bin/sleep +COPY --from=root-busybox /bin/dirname /bin/dirname + ENV USERNAME=nvs \ HOME=/home/nvs # DuckDB extension cache (aws, httpfs) for S3 log storage -COPY --chown=nvs:nvs --from=builder /root/.duckdb /root/.duckdb -COPY --chown=nvs:nvs --from=builder /root/.duckdb /home/nvs/.duckdb -COPY --chown=nvs:nvs --from=builder /app/.venv /app/.venv -ENV PATH="/app/.venv/bin:$PATH" -COPY --chown=nvs:nvs --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher +COPY --chown=1000:1000 --from=builder /root/.duckdb /root/.duckdb +COPY --chown=1000:1000 --from=builder /root/.duckdb /home/nvs/.duckdb +COPY --chown=1000:1000 --from=builder /app/.venv /app/.venv +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" \ + PYTHONPATH="/app/.venv/lib/python${PY_VERSION}/site-packages" +COPY --chown=1000:1000 --from=nmp-jobs-launcher /artifacts/jobs-launcher /tools/jobs-launcher ENTRYPOINT ["nemo", "services", "run"] EXPOSE 8080 USER 0 diff --git a/docker/Dockerfile.nmp-unsloth-training b/docker/Dockerfile.nmp-unsloth-training index 61729e72e3..f432ac6050 100644 --- a/docker/Dockerfile.nmp-unsloth-training +++ b/docker/Dockerfile.nmp-unsloth-training @@ -174,7 +174,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache \ --overrides /opt/docker/preserve_base_torch.txt \ - "aiohttp>=3.13.4,<4" \ + "aiohttp>=3.14.3,<4" \ "urllib3>=2.7.0,<3" \ "grpcio>=1.81.1,<2" \ "mistune>=3.3.3,<4" \ @@ -208,6 +208,11 @@ RUN rm -rf \ /usr/local/lib/python3.12/dist-packages/mlflow \ /usr/local/lib/python3.12/dist-packages/mlflow-*.dist-info \ /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/git \ + /usr/local/lib/python3.12/dist-packages/gitpython-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/GitPython-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/jupyterlab \ + /usr/local/lib/python3.12/dist-packages/jupyterlab-*.dist-info \ /usr/local/lib/python3.12/dist-packages/PIL \ /usr/local/lib/python3.12/dist-packages/pillow.libs \ /usr/local/lib/python3.12/dist-packages/pillow-*.dist-info \ diff --git a/docker/Dockerfile.safe-synthesizer-tasks b/docker/Dockerfile.safe-synthesizer-tasks index 159f80bf2a..8c59fa3d71 100644 --- a/docker/Dockerfile.safe-synthesizer-tasks +++ b/docker/Dockerfile.safe-synthesizer-tasks @@ -116,6 +116,7 @@ RUN printf '%s\n' \ > /tmp/safe-synthesizer-runtime.txt && \ printf '%s\n' \ wandb==0.28.1 \ + 'aiohttp>=3.14.3,<4' \ 'cryptography>=50.0.0,<51' \ 'pyarrow>=23.0.1,<24' \ > /tmp/safe-synthesizer-overrides.txt diff --git a/e2e/k8s/values/kind.yaml b/e2e/k8s/values/kind.yaml index c22548b34a..e4f5991add 100644 --- a/e2e/k8s/values/kind.yaml +++ b/e2e/k8s/values/kind.yaml @@ -7,6 +7,14 @@ postgresql: image: repository: docker.io/library/postgres +clickhouse: + persistence: + enabled: false + resources: + requests: + cpu: 500m + memory: 1Gi + core: storage: accessModes: diff --git a/packages/nmp_testing/src/nmp/testing/e2e/docker.py b/packages/nmp_testing/src/nmp/testing/e2e/docker.py index 42bf0ebac9..c81a54e56e 100644 --- a/packages/nmp_testing/src/nmp/testing/e2e/docker.py +++ b/packages/nmp_testing/src/nmp/testing/e2e/docker.py @@ -37,6 +37,12 @@ STARTUP_TIMEOUT_SECONDS = 60 NMP_API_NETWORK_ALIAS = "nmp-quickstart" NMP_API_CONTAINER_NAME_PREFIX = "nmp-api-test" +CLICKHOUSE_CONTAINER_PORT = 8123 +CLICKHOUSE_HEALTH_ENDPOINT = "/ping" +CLICKHOUSE_STARTUP_TIMEOUT_SECONDS = 60 +NMP_CLICKHOUSE_NETWORK_ALIAS = "nmp-intake-clickhouse" +NMP_CLICKHOUSE_CONTAINER_NAME_PREFIX = "nmp-clickhouse-test" +DEFAULT_E2E_CLICKHOUSE_IMAGE = "clickhouse/clickhouse-server:26.3" # Docker client timeout in seconds. This needs to be higher than the default 60s # to handle Docker-in-Docker (DinD) environments in CI where the Docker daemon @@ -49,6 +55,21 @@ def _api_container_name() -> str: return f"{NMP_API_CONTAINER_NAME_PREFIX}-{uuid.uuid4().hex[:8]}" +def _clickhouse_container_name() -> str: + """Return a readable, collision-resistant Docker container name for NeMo E2E.""" + return f"{NMP_CLICKHOUSE_CONTAINER_NAME_PREFIX}-{uuid.uuid4().hex[:8]}" + + +def _clickhouse_image() -> str: + """Return the ClickHouse image used by Docker E2E.""" + return os.environ.get("NMP_E2E_CLICKHOUSE_IMAGE", DEFAULT_E2E_CLICKHOUSE_IMAGE) + + +def _clickhouse_api_url() -> str: + """Return the ClickHouse URL reachable from sibling containers in the E2E network.""" + return f"http://{NMP_CLICKHOUSE_NETWORK_ALIAS}:{CLICKHOUSE_CONTAINER_PORT}" + + class Docker(E2EBackend): """Docker-based test backend using testcontainers. @@ -75,6 +96,7 @@ def __init__( gpu_requested=gpu_requested, ) self.container: DockerContainer | None = None + self.clickhouse_container: DockerContainer | None = None self.network: Network | None = None self._host_port: int | None = None self._data_dir: Path | None = None @@ -100,6 +122,8 @@ def start(self) -> None: self._data_dir = Path(tempfile.mkdtemp(prefix="nmp-e2e-data-")) logger.info(f"Created data directory: {self._data_dir}") + self._start_clickhouse_container() + # Create and configure the container # Use a longer timeout for DinD environments where Docker API calls can be slower self.container = DockerContainer( @@ -151,6 +175,9 @@ def start(self) -> None: self.container.with_env("NMP_SECRETS_ALLOW_KEY_CREATION", "1") # Enable mock provider mode for testing self.container.with_env("NMP_INFERENCE_GATEWAY_MOCK_PROVIDER_PREFIX", "igw-mock-") + # Intake runs inside the API container, so localhost-published ClickHouse + # ports are not reachable. Use the sidecar alias on the E2E network. + self.container.with_env("NMP_INTAKE_CLICKHOUSE_URL", _clickhouse_api_url()) # Ensure jobs use the same Docker network as the API for container execution self.container.with_env("NEMO_JOBS_DEFAULT_DOCKER_NETWORK", self.network.name) # Models Docker backend: use DonD mode so the API (in a container) can reach NIM @@ -234,6 +261,59 @@ def start(self) -> None: self.stop() raise + def _start_clickhouse_container(self) -> None: + """Start a ClickHouse sidecar reachable by the API container.""" + if self.network is None: + raise RuntimeError("Docker network must be created before ClickHouse") + + self.clickhouse_container = DockerContainer( + _clickhouse_image(), + docker_client_kw={"timeout": DOCKER_CLIENT_TIMEOUT_SECONDS}, + ) + self.clickhouse_container.with_kwargs(init=True) + self.clickhouse_container.with_name(_clickhouse_container_name()) + self.clickhouse_container.with_network(self.network) + self.clickhouse_container.with_network_aliases(NMP_CLICKHOUSE_NETWORK_ALIAS) + self.clickhouse_container.with_exposed_ports(CLICKHOUSE_CONTAINER_PORT) + self.clickhouse_container.with_env("CLICKHOUSE_USER", "default") + self.clickhouse_container.with_env("CLICKHOUSE_PASSWORD", "") + self.clickhouse_container.with_env("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1") + self.clickhouse_container.with_env("CLICKHOUSE_SKIP_USER_SETUP", "1") + + logger.info("Starting ClickHouse sidecar container: %s", _clickhouse_image()) + self.clickhouse_container.start() + self._wait_for_clickhouse_healthy() + + def _wait_for_clickhouse_healthy(self) -> None: + """Wait for the ClickHouse sidecar to accept HTTP requests.""" + if self.clickhouse_container is None: + raise RuntimeError("ClickHouse container was not created") + + container_host = self.clickhouse_container.get_container_host_ip() + host_port = int(self.clickhouse_container.get_exposed_port(CLICKHOUSE_CONTAINER_PORT)) + health_url = f"http://{container_host}:{host_port}{CLICKHOUSE_HEALTH_ENDPOINT}" + + logger.info(f"Waiting for ClickHouse health check at: {health_url}") + + start_time = time.time() + last_error: Exception | None = None + + while (time.time() - start_time) < CLICKHOUSE_STARTUP_TIMEOUT_SECONDS: + try: + response = httpx.get(health_url, timeout=5.0) + if response.status_code == 200: + return + last_error = Exception(f"ClickHouse health check returned status {response.status_code}") + except httpx.RequestError as e: + last_error = e + + time.sleep(1.0) + + msg = f"ClickHouse did not become healthy within {CLICKHOUSE_STARTUP_TIMEOUT_SECONDS}s" + if last_error: + msg += f": {last_error}" + raise RuntimeError(msg) + def _cleanup_existing_resources(self) -> None: """Clean up any lingering resources from a previous failed start. @@ -248,6 +328,14 @@ def _cleanup_existing_resources(self) -> None: logger.warning(f"Error stopping existing container: {e}") self.container = None + if self.clickhouse_container is not None: + logger.warning("Found existing ClickHouse container from previous attempt, cleaning up") + try: + self.clickhouse_container.stop() + except Exception as e: + logger.warning(f"Error stopping existing ClickHouse container: {e}") + self.clickhouse_container = None + if self.network is not None: logger.warning("Found existing network from previous attempt, cleaning up") try: @@ -356,12 +444,13 @@ def _collect_logs(self, log_dir: str = "docker/logs") -> None: and inspect output to log_dir. Safe to call in a partially degraded state — individual failures are ignored. """ - if self.container is None: + source_container = self.container or self.clickhouse_container + if source_container is None: return try: log_path = Path(log_dir) log_path.mkdir(parents=True, exist_ok=True) - docker_client = self.container.get_docker_client().client + docker_client = source_container.get_docker_client().client containers = docker_client.containers.list(all=True) for c in containers: name = c.name.lstrip("/") @@ -410,6 +499,28 @@ def stop(self) -> None: logger.warning(f"Error force removing container: {force_error}") self.container = None + if self.clickhouse_container: + container_id = None + try: + wrapped = self.clickhouse_container.get_wrapped_container() + container_id = wrapped.short_id + except Exception as e: + logger.debug(f"Could not get ClickHouse container ID for logging: {e}") + + try: + self.clickhouse_container.stop() + if container_id: + logger.info(f"Stopped ClickHouse container: {container_id}") + except Exception as e: + logger.warning(f"Error stopping ClickHouse container gracefully: {e}") + try: + wrapped = self.clickhouse_container.get_wrapped_container() + wrapped.remove(force=True) + logger.info(f"Force removed ClickHouse container: {container_id}") + except Exception as force_error: + logger.warning(f"Error force removing ClickHouse container: {force_error}") + self.clickhouse_container = None + if self.network: network_name = self.network.name try: diff --git a/packages/nmp_testing/src/nmp/testing/utils.py b/packages/nmp_testing/src/nmp/testing/utils.py index 064216ba49..69aea5e63e 100644 --- a/packages/nmp_testing/src/nmp/testing/utils.py +++ b/packages/nmp_testing/src/nmp/testing/utils.py @@ -30,6 +30,30 @@ _ENTITY_NAME_PATTERN = re.compile(NAME_PATTERN) +def _repo_venv_nemo_command(repo_root: Path) -> list[str] | None: + """Return the current repo venv's nemo console script when pytest already runs inside it.""" + venv = os.environ.get("VIRTUAL_ENV") + if not venv: + return None + + scripts_dir = "Scripts" if os.name == "nt" else "bin" + suffixes = (".exe", "") if os.name == "nt" else ("",) + for suffix in suffixes: + candidate = Path(venv) / scripts_dir / f"nemo{suffix}" + if not candidate.exists(): + continue + try: + candidate.resolve().relative_to(repo_root.resolve()) + except ValueError: + continue + return [str(candidate)] + return None + + +def _nemo_local_command(repo_root: Path) -> list[str]: + return _repo_venv_nemo_command(repo_root) or ["uv", "run", "--project", str(repo_root), "--frozen", "nemo"] + + def assert_exit_0(result: subprocess.CompletedProcess[str], msg: str) -> subprocess.CompletedProcess[str]: """Assert that a CLI invocation succeeded, including output in the failure message.""" assert result.returncode == 0, f"{msg}: {result.stderr or result.stdout}" @@ -67,21 +91,24 @@ def run_nemo_local( with a fake ``.git`` marker so ``skills install`` writes there instead of the real repo root). """ + repo_root = get_repo_root() with tempfile.TemporaryDirectory(prefix="nmp-cli-config-") as config_dir: env = os.environ.copy() config_path = Path(config_dir) / "config.yaml" config_path.write_text("{}\n") env["NMP_CONFIG_FILE"] = str(config_path) + if not env_extra or "NEMO_TELEMETRY_ENABLED" not in env_extra: + env["NEMO_TELEMETRY_ENABLED"] = "false" if base_url is not None: env["NMP_BASE_URL"] = base_url.rstrip("/") if workspace is not None: env["NMP_WORKSPACE"] = workspace if env_extra: env.update(env_extra) - cmd = ["uv", "run", "--project", str(get_repo_root()), "--frozen", "nemo", *args] + cmd = [*_nemo_local_command(repo_root), *args] return subprocess.run( cmd, - cwd=cwd or get_repo_root(), + cwd=cwd or repo_root, env=env, timeout=timeout, capture_output=True, diff --git a/packages/nmp_testing/tests/unit/test_e2e_docker_backend.py b/packages/nmp_testing/tests/unit/test_e2e_docker_backend.py new file mode 100644 index 0000000000..f7f709ea38 --- /dev/null +++ b/packages/nmp_testing/tests/unit/test_e2e_docker_backend.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the Docker E2E backend.""" + +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from nmp.testing.e2e import docker as docker_backend + + +class FakeDockerContainer: + instances: list["FakeDockerContainer"] = [] + start_order: list[str] = [] + + def __init__(self, image: str, **kwargs: Any): + self.image = image + self.kwargs = kwargs + self.name: str | None = None + self.network: object | None = None + self.aliases: list[str] = [] + self.exposed_ports: list[int] = [] + self.env: dict[str, str] = {} + self.volume_mappings: list[tuple[str, str, str]] = [] + self.container_kwargs: dict[str, Any] = {} + self.stopped = False + FakeDockerContainer.instances.append(self) + + def with_kwargs(self, **kwargs: Any) -> "FakeDockerContainer": + self.container_kwargs.update(kwargs) + return self + + def with_name(self, name: str) -> "FakeDockerContainer": + self.name = name + return self + + def with_network(self, network: object) -> "FakeDockerContainer": + self.network = network + return self + + def with_network_aliases(self, *aliases: str) -> "FakeDockerContainer": + self.aliases.extend(aliases) + return self + + def with_exposed_ports(self, *ports: int) -> "FakeDockerContainer": + self.exposed_ports.extend(ports) + return self + + def with_volume_mapping(self, source: str, target: str, mode: str) -> "FakeDockerContainer": + self.volume_mappings.append((source, target, mode)) + return self + + def with_env(self, key: str, value: str) -> "FakeDockerContainer": + self.env[key] = value + return self + + def start(self) -> None: + assert self.name is not None + FakeDockerContainer.start_order.append(self.name) + + def stop(self) -> None: + self.stopped = True + + def get_wrapped_container(self) -> object: + return SimpleNamespace( + status="running", + short_id=self.name, + reload=lambda: None, + remove=lambda **_kwargs: None, + ) + + def get_exposed_port(self, _port: int) -> str: + return "32768" + + def get_container_host_ip(self) -> str: + return "localhost" + + +class FakeNetwork: + def __init__(self): + self.name = "nmp-e2e-test-network" + self.created = False + self.removed = False + + def create(self) -> None: + self.created = True + + def remove(self) -> None: + self.removed = True + + +def test_docker_backend_starts_clickhouse_sidecar_on_api_network( + monkeypatch, + tmp_path: Path, +) -> None: + config_path = tmp_path / "config.yaml" + config_path.write_text("platform: {}\n") + FakeDockerContainer.instances = [] + FakeDockerContainer.start_order = [] + + monkeypatch.setattr(docker_backend, "DockerContainer", FakeDockerContainer) + monkeypatch.setattr(docker_backend, "Network", FakeNetwork) + monkeypatch.setattr(docker_backend.Docker, "_wait_for_clickhouse_healthy", lambda _self: None) + monkeypatch.setattr(docker_backend.Docker, "_wait_for_healthy", lambda _self: None) + monkeypatch.setattr(docker_backend.Docker, "_collect_logs", lambda _self: None) + + backend = docker_backend.Docker(config_path, registry="registry.example/nmp", tag="test-tag") + + try: + backend.start() + + clickhouse_container, api_container = FakeDockerContainer.instances + assert clickhouse_container.image == docker_backend.DEFAULT_E2E_CLICKHOUSE_IMAGE + assert clickhouse_container.network is backend.network + assert clickhouse_container.aliases == [docker_backend.NMP_CLICKHOUSE_NETWORK_ALIAS] + assert clickhouse_container.env["CLICKHOUSE_SKIP_USER_SETUP"] == "1" + + assert api_container.network is backend.network + assert api_container.env["NMP_INTAKE_CLICKHOUSE_URL"] == docker_backend._clickhouse_api_url() + assert api_container.env["NEMO_JOBS_DEFAULT_DOCKER_NETWORK"] == "nmp-e2e-test-network" + assert api_container.env["NMP_IMAGE_REGISTRY"] == "registry.example/nmp" + assert api_container.env["NMP_IMAGE_TAG"] == "test-tag" + + assert FakeDockerContainer.start_order == [ + clickhouse_container.name, + api_container.name, + ] + finally: + backend.stop() + + +def test_clickhouse_image_can_be_overridden(monkeypatch) -> None: + monkeypatch.setenv("NMP_E2E_CLICKHOUSE_IMAGE", "example/clickhouse:test") + + assert docker_backend._clickhouse_image() == "example/clickhouse:test" diff --git a/packages/nmp_testing/tests/unit/test_utils.py b/packages/nmp_testing/tests/unit/test_utils.py index c16a064506..73e99673ba 100644 --- a/packages/nmp_testing/tests/unit/test_utils.py +++ b/packages/nmp_testing/tests/unit/test_utils.py @@ -3,9 +3,14 @@ """Unit tests for testing utils (e.g. short_unique_name, add_mock_provider validation).""" +import os import re +import subprocess +from pathlib import Path +from typing import Any from nmp.common.entities.constants import NAME_PATTERN +from nmp.testing import utils from nmp.testing.utils import short_unique_name _ENTITY_NAME_PATTERN = re.compile(NAME_PATTERN) @@ -41,3 +46,63 @@ def test_consecutive_hyphens_collapsed(self): name = short_unique_name("a--b") assert "--" not in name assert _ENTITY_NAME_PATTERN.match(name) + + +class TestRunNemoLocal: + def test_uses_repo_virtualenv_nemo_when_available(self, tmp_path: Path, monkeypatch): + repo_root = tmp_path / "repo" + nemo = repo_root / ".venv" / ("Scripts" if os.name == "nt" else "bin") / "nemo" + nemo.parent.mkdir(parents=True) + nemo.write_text("#!/bin/sh\n") + monkeypatch.setenv("VIRTUAL_ENV", str(repo_root / ".venv")) + monkeypatch.setattr(utils, "get_repo_root", lambda: repo_root) + + calls: list[dict[str, Any]] = [] + + def fake_run(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + calls.append({"cmd": cmd, **kwargs}) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + monkeypatch.setattr(utils.subprocess, "run", fake_run) + + result = utils.run_nemo_local("config", "current-context") + + assert result.returncode == 0 + assert calls[0]["cmd"] == [str(nemo), "config", "current-context"] + assert calls[0]["env"]["NEMO_TELEMETRY_ENABLED"] == "false" + + def test_falls_back_to_uv_when_not_running_from_repo_virtualenv(self, tmp_path: Path, monkeypatch): + repo_root = tmp_path / "repo" + repo_root.mkdir() + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.setattr(utils, "get_repo_root", lambda: repo_root) + + calls: list[dict[str, Any]] = [] + + def fake_run(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + calls.append({"cmd": cmd, **kwargs}) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + monkeypatch.setattr(utils.subprocess, "run", fake_run) + + utils.run_nemo_local("config", "view") + + assert calls[0]["cmd"] == ["uv", "run", "--project", str(repo_root), "--frozen", "nemo", "config", "view"] + + def test_env_extra_can_enable_telemetry(self, tmp_path: Path, monkeypatch): + repo_root = tmp_path / "repo" + repo_root.mkdir() + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.setattr(utils, "get_repo_root", lambda: repo_root) + + calls: list[dict[str, Any]] = [] + + def fake_run(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + calls.append({"cmd": cmd, **kwargs}) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + monkeypatch.setattr(utils.subprocess, "run", fake_run) + + utils.run_nemo_local("config", "view", env_extra={"NEMO_TELEMETRY_ENABLED": "true"}) + + assert calls[0]["env"]["NEMO_TELEMETRY_ENABLED"] == "true"